Examples with ChecksumType org.apache.hadoop.hbase.util.ChecksumType used on opensource projects

Example 1 with ChecksumType

use of org.apache.hadoop.hbase.util.ChecksumType in project hbase by apache.

the class ChecksumUtil method validateChecksum.

/**
   * Validates that the data in the specified HFileBlock matches the checksum. Generates the
   * checksums for the data and then validate that it matches those stored in the end of the data.
   * @param buffer Contains the data in following order: HFileBlock header, data, checksums.
   * @param pathName Path of the HFile to which the {@code data} belongs. Only used for logging.
   * @param offset offset of the data being validated. Only used for logging.
   * @param hdrSize Size of the block header in {@code data}. Only used for logging.
   * @return True if checksum matches, else false.
   */
static boolean validateChecksum(ByteBuffer buffer, String pathName, long offset, int hdrSize) throws IOException {
    // A ChecksumType.NULL indicates that the caller is not interested in validating checksums,
    // so we always return true.
    ChecksumType cktype = ChecksumType.codeToType(buffer.get(HFileBlock.Header.CHECKSUM_TYPE_INDEX));
    if (cktype == ChecksumType.NULL) {
        // No checksum validations needed for this block.
        return true;
    }
    // read in the stored value of the checksum size from the header.
    int bytesPerChecksum = buffer.getInt(HFileBlock.Header.BYTES_PER_CHECKSUM_INDEX);
    DataChecksum dataChecksum = DataChecksum.newDataChecksum(cktype.getDataChecksumType(), bytesPerChecksum);
    assert dataChecksum != null;
    int onDiskDataSizeWithHeader = buffer.getInt(HFileBlock.Header.ON_DISK_DATA_SIZE_WITH_HEADER_INDEX);
    if (LOG.isTraceEnabled()) {
        LOG.info("dataLength=" + buffer.capacity() + ", sizeWithHeader=" + onDiskDataSizeWithHeader + ", checksumType=" + cktype.getName() + ", file=" + pathName + ", offset=" + offset + ", headerSize=" + hdrSize + ", bytesPerChecksum=" + bytesPerChecksum);
    }
    try {
        ByteBuffer data = (ByteBuffer) buffer.duplicate().position(0).limit(onDiskDataSizeWithHeader);
        ByteBuffer checksums = (ByteBuffer) buffer.duplicate().position(onDiskDataSizeWithHeader).limit(buffer.capacity());
        dataChecksum.verifyChunkedSums(data, checksums, pathName, 0);
    } catch (ChecksumException e) {
        return false;
    }
    // checksum is valid
    return true;
}

Example 2 with ChecksumType

use of org.apache.hadoop.hbase.util.ChecksumType in project hbase by apache.

the class TestChecksum method testAllChecksumTypes.

/**
   * Test all checksum types by writing and reading back blocks.
   */
@Test
public void testAllChecksumTypes() throws IOException {
    List<ChecksumType> cktypes = new ArrayList<>(Arrays.asList(ChecksumType.values()));
    for (Iterator<ChecksumType> itr = cktypes.iterator(); itr.hasNext(); ) {
        ChecksumType cktype = itr.next();
        Path path = new Path(TEST_UTIL.getDataTestDir(), "checksum" + cktype.getName());
        FSDataOutputStream os = fs.create(path);
        HFileContext meta = new HFileContextBuilder().withChecksumType(cktype).build();
        HFileBlock.Writer hbw = new HFileBlock.Writer(null, meta);
        DataOutputStream dos = hbw.startWriting(BlockType.DATA);
        for (int i = 0; i < 1000; ++i) {
            dos.writeInt(i);
        }
        hbw.writeHeaderAndData(os);
        int totalSize = hbw.getOnDiskSizeWithHeader();
        os.close();
        // Use hbase checksums.
        assertEquals(true, hfs.useHBaseChecksum());
        FSDataInputStreamWrapper is = new FSDataInputStreamWrapper(fs, path);
        meta = new HFileContextBuilder().withHBaseCheckSum(true).build();
        HFileBlock.FSReader hbr = new HFileBlock.FSReaderImpl(is, totalSize, (HFileSystem) fs, path, meta);
        HFileBlock b = hbr.readBlockData(0, -1, false);
        ByteBuff data = b.getBufferWithoutHeader();
        for (int i = 0; i < 1000; i++) {
            assertEquals(i, data.getInt());
        }
        boolean exception_thrown = false;
        try {
            data.getInt();
        } catch (BufferUnderflowException e) {
            exception_thrown = true;
        }
        assertTrue(exception_thrown);
        assertEquals(0, HFile.getChecksumFailuresCount());
    }
}

Example 3 with ChecksumType

use of org.apache.hadoop.hbase.util.ChecksumType in project hbase by apache.

the class ChecksumUtil method validateChecksum.

/**
 * Validates that the data in the specified HFileBlock matches the checksum. Generates the
 * checksums for the data and then validate that it matches those stored in the end of the data.
 * @param buf Contains the data in following order: HFileBlock header, data, checksums.
 * @param pathName Path of the HFile to which the {@code data} belongs. Only used for logging.
 * @param offset offset of the data being validated. Only used for logging.
 * @param hdrSize Size of the block header in {@code data}. Only used for logging.
 * @return True if checksum matches, else false.
 */
static boolean validateChecksum(ByteBuff buf, String pathName, long offset, int hdrSize) {
    ChecksumType ctype = ChecksumType.codeToType(buf.get(HFileBlock.Header.CHECKSUM_TYPE_INDEX));
    if (ctype == ChecksumType.NULL) {
        // No checksum validations needed for this block.
        return true;
    }
    // read in the stored value of the checksum size from the header.
    int bytesPerChecksum = buf.getInt(HFileBlock.Header.BYTES_PER_CHECKSUM_INDEX);
    DataChecksum dataChecksum = DataChecksum.newDataChecksum(ctype.getDataChecksumType(), bytesPerChecksum);
    assert dataChecksum != null;
    int onDiskDataSizeWithHeader = buf.getInt(HFileBlock.Header.ON_DISK_DATA_SIZE_WITH_HEADER_INDEX);
    LOG.trace("dataLength={}, sizeWithHeader={}, checksumType={}, file={}, " + "offset={}, headerSize={}, bytesPerChecksum={}", buf.capacity(), onDiskDataSizeWithHeader, ctype.getName(), pathName, offset, hdrSize, bytesPerChecksum);
    ByteBuff data = buf.duplicate().position(0).limit(onDiskDataSizeWithHeader);
    ByteBuff checksums = buf.duplicate().position(onDiskDataSizeWithHeader).limit(buf.limit());
    return verifyChunkedSums(dataChecksum, data, checksums, pathName);
}

Example 4 with ChecksumType

use of org.apache.hadoop.hbase.util.ChecksumType in project hbase by apache.

the class TestChecksum method testVerifyCheckSum.

@Test
public void testVerifyCheckSum() throws IOException {
    int intCount = 10000;
    for (ChecksumType ckt : ChecksumType.values()) {
        Path path = new Path(TEST_UTIL.getDataTestDir(), "checksum" + ckt.getName());
        FSDataOutputStream os = fs.create(path);
        HFileContext meta = new HFileContextBuilder().withChecksumType(ckt).build();
        HFileBlock.Writer hbw = new HFileBlock.Writer(TEST_UTIL.getConfiguration(), null, meta);
        DataOutputStream dos = hbw.startWriting(BlockType.DATA);
        for (int i = 0; i < intCount; ++i) {
            dos.writeInt(i);
        }
        hbw.writeHeaderAndData(os);
        int totalSize = hbw.getOnDiskSizeWithHeader();
        os.close();
        // Use hbase checksums.
        assertEquals(true, hfs.useHBaseChecksum());
        FSDataInputStreamWrapper is = new FSDataInputStreamWrapper(fs, path);
        meta = new HFileContextBuilder().withHBaseCheckSum(true).build();
        ReaderContext context = new ReaderContextBuilder().withInputStreamWrapper(is).withFileSize(totalSize).withFileSystem((HFileSystem) fs).withFilePath(path).build();
        HFileBlock.FSReader hbr = new HFileBlock.FSReaderImpl(context, meta, ByteBuffAllocator.HEAP, TEST_UTIL.getConfiguration());
        HFileBlock b = hbr.readBlockData(0, -1, false, false, true);
        assertTrue(!b.isSharedMem());
        // verify SingleByteBuff checksum.
        verifySBBCheckSum(b.getBufferReadOnly());
        // verify MultiByteBuff checksum.
        verifyMBBCheckSum(b.getBufferReadOnly());
        ByteBuff data = b.getBufferWithoutHeader();
        for (int i = 0; i < intCount; i++) {
            assertEquals(i, data.getInt());
        }
        try {
            data.getInt();
            fail();
        } catch (BufferUnderflowException e) {
        // expected failure
        }
        assertEquals(0, HFile.getAndResetChecksumFailuresCount());
    }
}