Examples with FSDataInputStreamWrapper org.apache.hadoop.hbase.io.FSDataInputStreamWrapper used on opensource projects

Example 11 with FSDataInputStreamWrapper

use of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper in project hbase by apache.

the class HFilePrettyPrinter method processFile.

// HBASE-22561 introduces boolean checkRootDir for WebUI specificly
public int processFile(Path file, boolean checkRootDir) throws IOException {
    if (verbose) {
        out.println("Scanning -> " + file);
    }
    if (checkRootDir) {
        Path rootPath = CommonFSUtils.getRootDir(getConf());
        String rootString = rootPath + Path.SEPARATOR;
        if (!file.toString().startsWith(rootString)) {
            // First we see if fully-qualified URI matches the root dir. It might
            // also be an absolute path in the same filesystem, so we prepend the FS
            // of the root dir and see if that fully-qualified URI matches.
            FileSystem rootFS = rootPath.getFileSystem(getConf());
            String qualifiedFile = rootFS.getUri().toString() + file.toString();
            if (!qualifiedFile.startsWith(rootString)) {
                err.println("ERROR, file (" + file + ") is not in HBase's root directory (" + rootString + ")");
                return -2;
            }
        }
    }
    FileSystem fs = file.getFileSystem(getConf());
    if (!fs.exists(file)) {
        err.println("ERROR, file doesnt exist: " + file);
        return -2;
    }
    HFile.Reader reader = HFile.createReader(fs, file, CacheConfig.DISABLED, true, getConf());
    Map<byte[], byte[]> fileInfo = reader.getHFileInfo();
    KeyValueStatsCollector fileStats = null;
    if (verbose || printKey || checkRow || checkFamily || printStats || checkMobIntegrity) {
        // scan over file and read key/value's and check if requested
        HFileScanner scanner = reader.getScanner(getConf(), false, false, false);
        fileStats = new KeyValueStatsCollector();
        boolean shouldScanKeysValues;
        if (this.isSeekToRow && !Bytes.equals(row, reader.getFirstRowKey().orElse(null))) {
            // seek to the first kv on this row
            shouldScanKeysValues = (scanner.seekTo(PrivateCellUtil.createFirstOnRow(this.row)) != -1);
        } else {
            shouldScanKeysValues = scanner.seekTo();
        }
        if (shouldScanKeysValues) {
            scanKeysValues(file, fileStats, scanner, row);
        }
    }
    // print meta data
    if (shouldPrintMeta) {
        printMeta(reader, fileInfo);
    }
    if (printBlockIndex) {
        out.println("Block Index:");
        out.println(reader.getDataBlockIndexReader());
    }
    if (printBlockHeaders) {
        out.println("Block Headers:");
        /*
       * TODO: this same/similar block iteration logic is used in HFileBlock#blockRange and
       * TestLazyDataBlockDecompression. Refactor?
       */
        FSDataInputStreamWrapper fsdis = new FSDataInputStreamWrapper(fs, file);
        long fileSize = fs.getFileStatus(file).getLen();
        FixedFileTrailer trailer = FixedFileTrailer.readFromStream(fsdis.getStream(false), fileSize);
        long offset = trailer.getFirstDataBlockOffset(), max = trailer.getLastDataBlockOffset();
        HFileBlock block;
        while (offset <= max) {
            block = reader.readBlock(offset, -1, /* cacheBlock */
            false, /* pread */
            false, /* isCompaction */
            false, /* updateCacheMetrics */
            false, null, null);
            offset += block.getOnDiskSizeWithHeader();
            out.println(block);
        }
    }
    if (printStats) {
        fileStats.finish();
        out.println("Stats:\n" + fileStats);
    }
    reader.close();
    return 0;
}

Example 12 with FSDataInputStreamWrapper

use of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper in project hbase by apache.

the class HFile method isHFileFormat.

/**
 * Returns true if the specified file has a valid HFile Trailer.
 * @param fs filesystem
 * @param fileStatus the file to verify
 * @return true if the file has a valid HFile Trailer, otherwise false
 * @throws IOException if failed to read from the underlying stream
 */
public static boolean isHFileFormat(final FileSystem fs, final FileStatus fileStatus) throws IOException {
    final Path path = fileStatus.getPath();
    final long size = fileStatus.getLen();
    try (FSDataInputStreamWrapper fsdis = new FSDataInputStreamWrapper(fs, path)) {
        boolean isHBaseChecksum = fsdis.shouldUseHBaseChecksum();
        // Initially we must read with FS checksum.
        assert !isHBaseChecksum;
        FixedFileTrailer.readFromStream(fsdis.getStream(isHBaseChecksum), size);
        return true;
    } catch (IllegalArgumentException e) {
        return false;
    }
}

Example 13 with FSDataInputStreamWrapper

use of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper in project hbase by apache.

the class StoreFileInfo method createReaderContext.

ReaderContext createReaderContext(boolean doDropBehind, long readahead, ReaderType type) throws IOException {
    FSDataInputStreamWrapper in;
    FileStatus status;
    if (this.link != null) {
        // HFileLink
        in = new FSDataInputStreamWrapper(fs, this.link, doDropBehind, readahead);
        status = this.link.getFileStatus(fs);
    } else if (this.reference != null) {
        // HFile Reference
        Path referencePath = getReferredToFile(this.getPath());
        try {
            in = new FSDataInputStreamWrapper(fs, referencePath, doDropBehind, readahead);
        } catch (FileNotFoundException fnfe) {
            // Intercept the exception so can insert more info about the Reference; otherwise
            // exception just complains about some random file -- operator doesn't realize it
            // other end of a Reference
            FileNotFoundException newFnfe = new FileNotFoundException(toString());
            newFnfe.initCause(fnfe);
            throw newFnfe;
        }
        status = fs.getFileStatus(referencePath);
    } else {
        in = new FSDataInputStreamWrapper(fs, this.getPath(), doDropBehind, readahead);
        status = fs.getFileStatus(initialPath);
    }
    long length = status.getLen();
    ReaderContextBuilder contextBuilder = new ReaderContextBuilder().withInputStreamWrapper(in).withFileSize(length).withPrimaryReplicaReader(this.primaryReplica).withReaderType(type).withFileSystem(fs);
    if (this.reference != null) {
        contextBuilder.withFilePath(this.getPath());
    } else {
        contextBuilder.withFilePath(status.getPath());
    }
    return contextBuilder.build();
}

Example 14 with FSDataInputStreamWrapper

use of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper in project hbase by apache.

the class TestHFileWriterV3WithDataEncoders method writeDataAndReadFromHFile.

private void writeDataAndReadFromHFile(Path hfilePath, Compression.Algorithm compressAlgo, int entryCount, boolean findMidKey, boolean useTags) throws IOException {
    HFileContext context = new HFileContextBuilder().withBlockSize(4096).withIncludesTags(useTags).withDataBlockEncoding(dataBlockEncoding).withCellComparator(CellComparatorImpl.COMPARATOR).withCompression(compressAlgo).build();
    CacheConfig cacheConfig = new CacheConfig(conf);
    HFile.Writer writer = new HFile.WriterFactory(conf, cacheConfig).withPath(fs, hfilePath).withFileContext(context).create();
    // Just a fixed seed.
    Random rand = new Random(9713312);
    List<KeyValue> keyValues = new ArrayList<>(entryCount);
    writeKeyValues(entryCount, useTags, writer, rand, keyValues);
    FSDataInputStream fsdis = fs.open(hfilePath);
    long fileSize = fs.getFileStatus(hfilePath).getLen();
    FixedFileTrailer trailer = FixedFileTrailer.readFromStream(fsdis, fileSize);
    Assert.assertEquals(3, trailer.getMajorVersion());
    Assert.assertEquals(entryCount, trailer.getEntryCount());
    HFileContext meta = new HFileContextBuilder().withCompression(compressAlgo).withIncludesMvcc(true).withIncludesTags(useTags).withDataBlockEncoding(dataBlockEncoding).withHBaseCheckSum(true).build();
    ReaderContext readerContext = new ReaderContextBuilder().withInputStreamWrapper(new FSDataInputStreamWrapper(fsdis)).withFilePath(hfilePath).withFileSystem(fs).withFileSize(fileSize).build();
    HFileBlock.FSReader blockReader = new HFileBlock.FSReaderImpl(readerContext, meta, ByteBuffAllocator.HEAP, conf);
    // Comparator class name is stored in the trailer in version 3.
    CellComparator comparator = trailer.createComparator();
    HFileBlockIndex.BlockIndexReader dataBlockIndexReader = new HFileBlockIndex.CellBasedKeyBlockIndexReader(comparator, trailer.getNumDataIndexLevels());
    HFileBlockIndex.BlockIndexReader metaBlockIndexReader = new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1);
    HFileBlock.BlockIterator blockIter = blockReader.blockRange(trailer.getLoadOnOpenDataOffset(), fileSize - trailer.getTrailerSize());
    // Data index. We also read statistics about the block index written after
    // the root level.
    dataBlockIndexReader.readMultiLevelIndexRoot(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), trailer.getDataIndexCount());
    FSDataInputStreamWrapper wrapper = new FSDataInputStreamWrapper(fs, hfilePath);
    readerContext = new ReaderContextBuilder().withFilePath(hfilePath).withFileSize(fileSize).withFileSystem(wrapper.getHfs()).withInputStreamWrapper(wrapper).build();
    HFileInfo hfile = new HFileInfo(readerContext, conf);
    HFile.Reader reader = new HFilePreadReader(readerContext, hfile, cacheConfig, conf);
    hfile.initMetaAndIndex(reader);
    if (findMidKey) {
        Cell midkey = dataBlockIndexReader.midkey(reader);
        Assert.assertNotNull("Midkey should not be null", midkey);
    }
    // Meta index.
    metaBlockIndexReader.readRootIndex(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX).getByteStream(), trailer.getMetaIndexCount());
    // File info
    HFileInfo fileInfo = new HFileInfo();
    fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
    byte[] keyValueFormatVersion = fileInfo.get(HFileWriterImpl.KEY_VALUE_VERSION);
    boolean includeMemstoreTS = keyValueFormatVersion != null && Bytes.toInt(keyValueFormatVersion) > 0;
    // Counters for the number of key/value pairs and the number of blocks
    int entriesRead = 0;
    int blocksRead = 0;
    long memstoreTS = 0;
    DataBlockEncoder encoder = dataBlockEncoding.getEncoder();
    long curBlockPos = scanBlocks(entryCount, context, keyValues, fsdis, trailer, meta, blockReader, entriesRead, blocksRead, encoder);
    // Meta blocks. We can scan until the load-on-open data offset (which is
    // the root block index offset in version 2) because we are not testing
    // intermediate-level index blocks here.
    int metaCounter = 0;
    while (fsdis.getPos() < trailer.getLoadOnOpenDataOffset()) {
        LOG.info("Current offset: {}, scanning until {}", fsdis.getPos(), trailer.getLoadOnOpenDataOffset());
        HFileBlock block = blockReader.readBlockData(curBlockPos, -1, false, false, true).unpack(context, blockReader);
        Assert.assertEquals(BlockType.META, block.getBlockType());
        Text t = new Text();
        ByteBuff buf = block.getBufferWithoutHeader();
        if (Writables.getWritable(buf.array(), buf.arrayOffset(), buf.limit(), t) == null) {
            throw new IOException("Failed to deserialize block " + this + " into a " + t.getClass().getSimpleName());
        }
        Text expectedText = (metaCounter == 0 ? new Text("Paris") : metaCounter == 1 ? new Text("Moscow") : new Text("Washington, D.C."));
        Assert.assertEquals(expectedText, t);
        LOG.info("Read meta block data: " + t);
        ++metaCounter;
        curBlockPos += block.getOnDiskSizeWithHeader();
    }
    fsdis.close();
    reader.close();
}

Example 15 with FSDataInputStreamWrapper

use of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper in project hbase by apache.

the class TestHFileBlock method testReaderV2Internals.

protected void testReaderV2Internals() throws IOException {
    final Configuration conf = TEST_UTIL.getConfiguration();
    if (includesTag) {
        conf.setInt("hfile.format.version", 3);
    }
    for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
        for (boolean pread : new boolean[] { false, true }) {
            LOG.info("testReaderV2: Compression algorithm: " + algo + ", pread=" + pread);
            Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_" + algo);
            FSDataOutputStream os = fs.create(path);
            HFileContext meta = new HFileContextBuilder().withCompression(algo).withIncludesMvcc(includesMemstoreTS).withIncludesTags(includesTag).withBytesPerCheckSum(HFile.DEFAULT_BYTES_PER_CHECKSUM).build();
            HFileBlock.Writer hbw = new HFileBlock.Writer(conf, null, meta);
            long totalSize = 0;
            for (int blockId = 0; blockId < 2; ++blockId) {
                DataOutputStream dos = hbw.startWriting(BlockType.DATA);
                for (int i = 0; i < 1234; ++i) dos.writeInt(i);
                hbw.writeHeaderAndData(os);
                totalSize += hbw.getOnDiskSizeWithHeader();
            }
            os.close();
            FSDataInputStream is = fs.open(path);
            meta = new HFileContextBuilder().withHBaseCheckSum(true).withIncludesMvcc(includesMemstoreTS).withIncludesTags(includesTag).withCompression(algo).build();
            ReaderContext context = new ReaderContextBuilder().withInputStreamWrapper(new FSDataInputStreamWrapper(is)).withFileSize(totalSize).withFilePath(path).withFileSystem(fs).build();
            HFileBlock.FSReader hbr = new HFileBlock.FSReaderImpl(context, meta, alloc, TEST_UTIL.getConfiguration());
            HFileBlock b = hbr.readBlockData(0, -1, pread, false, true);
            is.close();
            assertEquals(0, HFile.getAndResetChecksumFailuresCount());
            b.sanityCheck();
            assertEquals(4936, b.getUncompressedSizeWithoutHeader());
            assertEquals(algo == GZ ? 2173 : 4936, b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
            HFileBlock expected = b;
            if (algo == GZ) {
                is = fs.open(path);
                ReaderContext readerContext = new ReaderContextBuilder().withInputStreamWrapper(new FSDataInputStreamWrapper(is)).withFileSize(totalSize).withFilePath(path).withFileSystem(fs).build();
                hbr = new HFileBlock.FSReaderImpl(readerContext, meta, alloc, TEST_UTIL.getConfiguration());
                b = hbr.readBlockData(0, 2173 + HConstants.HFILEBLOCK_HEADER_SIZE + b.totalChecksumBytes(), pread, false, true);
                assertEquals(expected, b);
                int wrongCompressedSize = 2172;
                try {
                    hbr.readBlockData(0, wrongCompressedSize + HConstants.HFILEBLOCK_HEADER_SIZE, pread, false, true);
                    fail("Exception expected");
                } catch (IOException ex) {
                    String expectedPrefix = "Passed in onDiskSizeWithHeader=";
                    assertTrue("Invalid exception message: '" + ex.getMessage() + "'.\nMessage is expected to start with: '" + expectedPrefix + "'", ex.getMessage().startsWith(expectedPrefix));
                }
                assertRelease(b);
                is.close();
            }
            assertRelease(expected);
        }
    }
}