Examples with Checksum java.util.zip.Checksum used on opensource projects

Example 21 with Checksum

use of java.util.zip.Checksum in project pinot by linkedin.

the class CrcUtils method computeCrc.

public long computeCrc() {
    CheckedInputStream cis = null;
    InputStream is = null;
    final Checksum checksum = new Adler32();
    final byte[] tempBuf = new byte[BUFFER_SIZE];
    for (final File file : filesToProcess) {
        try {
            is = new BufferedInputStream(new FileInputStream(file));
            cis = new CheckedInputStream(is, checksum);
            while (cis.read(tempBuf) >= 0) {
            }
            cis.close();
            is.close();
        } catch (final Exception e) {
            LOGGER.error("Caught exception while computing CRC", e);
            Utils.rethrowException(e);
            throw new AssertionError("Should not reach this");
        }
    }
    return checksum.getValue();
}

Example 22 with Checksum

use of java.util.zip.Checksum in project netty by netty.

the class FastLzFrameDecoder method decode.

@Override
protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
    try {
        switch(currentState) {
            case INIT_BLOCK:
                if (in.readableBytes() < 4) {
                    break;
                }
                final int magic = in.readUnsignedMedium();
                if (magic != MAGIC_NUMBER) {
                    throw new DecompressionException("unexpected block identifier");
                }
                final byte options = in.readByte();
                isCompressed = (options & 0x01) == BLOCK_TYPE_COMPRESSED;
                hasChecksum = (options & 0x10) == BLOCK_WITH_CHECKSUM;
                currentState = State.INIT_BLOCK_PARAMS;
            case INIT_BLOCK_PARAMS:
                if (in.readableBytes() < 2 + (isCompressed ? 2 : 0) + (hasChecksum ? 4 : 0)) {
                    break;
                }
                currentChecksum = hasChecksum ? in.readInt() : 0;
                chunkLength = in.readUnsignedShort();
                originalLength = isCompressed ? in.readUnsignedShort() : chunkLength;
                currentState = State.DECOMPRESS_DATA;
            case DECOMPRESS_DATA:
                final int chunkLength = this.chunkLength;
                if (in.readableBytes() < chunkLength) {
                    break;
                }
                final int idx = in.readerIndex();
                final int originalLength = this.originalLength;
                final ByteBuf uncompressed;
                final byte[] output;
                final int outputPtr;
                if (originalLength != 0) {
                    uncompressed = ctx.alloc().heapBuffer(originalLength, originalLength);
                    output = uncompressed.array();
                    outputPtr = uncompressed.arrayOffset() + uncompressed.writerIndex();
                } else {
                    uncompressed = null;
                    output = EmptyArrays.EMPTY_BYTES;
                    outputPtr = 0;
                }
                boolean success = false;
                try {
                    if (isCompressed) {
                        final byte[] input;
                        final int inputPtr;
                        if (in.hasArray()) {
                            input = in.array();
                            inputPtr = in.arrayOffset() + idx;
                        } else {
                            input = new byte[chunkLength];
                            in.getBytes(idx, input);
                            inputPtr = 0;
                        }
                        final int decompressedBytes = decompress(input, inputPtr, chunkLength, output, outputPtr, originalLength);
                        if (originalLength != decompressedBytes) {
                            throw new DecompressionException(String.format("stream corrupted: originalLength(%d) and actual length(%d) mismatch", originalLength, decompressedBytes));
                        }
                    } else {
                        in.getBytes(idx, output, outputPtr, chunkLength);
                    }
                    final Checksum checksum = this.checksum;
                    if (hasChecksum && checksum != null) {
                        checksum.reset();
                        checksum.update(output, outputPtr, originalLength);
                        final int checksumResult = (int) checksum.getValue();
                        if (checksumResult != currentChecksum) {
                            throw new DecompressionException(String.format("stream corrupted: mismatching checksum: %d (expected: %d)", checksumResult, currentChecksum));
                        }
                    }
                    if (uncompressed != null) {
                        uncompressed.writerIndex(uncompressed.writerIndex() + originalLength);
                        out.add(uncompressed);
                    }
                    in.skipBytes(chunkLength);
                    currentState = State.INIT_BLOCK;
                    success = true;
                } finally {
                    if (!success) {
                        uncompressed.release();
                    }
                }
                break;
            case CORRUPTED:
                in.skipBytes(in.readableBytes());
                break;
            default:
                throw new IllegalStateException();
        }
    } catch (Exception e) {
        currentState = State.CORRUPTED;
        throw e;
    }
}

Example 23 with Checksum

use of java.util.zip.Checksum in project netty by netty.

the class Lz4FrameEncoderTest method newEncoder.

private Lz4FrameEncoder newEncoder(int blockSize, int maxEncodeSize) {
    Checksum checksum = XXHashFactory.fastestInstance().newStreamingHash32(DEFAULT_SEED).asChecksum();
    Lz4FrameEncoder encoder = new Lz4FrameEncoder(LZ4Factory.fastestInstance(), true, blockSize, checksum, maxEncodeSize);
    encoder.handlerAdded(ctx);
    return encoder;
}

Example 24 with Checksum

use of java.util.zip.Checksum in project hadoop by apache.

the class BlockReceiver method computePartialChunkCrc.

/**
   * reads in the partial crc chunk and computes checksum
   * of pre-existing data in partial chunk.
   */
private Checksum computePartialChunkCrc(long blkoff, long ckoff) throws IOException {
    // find offset of the beginning of partial chunk.
    //
    int sizePartialChunk = (int) (blkoff % bytesPerChecksum);
    blkoff = blkoff - sizePartialChunk;
    if (LOG.isDebugEnabled()) {
        LOG.debug("computePartialChunkCrc for " + block + ": sizePartialChunk=" + sizePartialChunk + ", block offset=" + blkoff + ", metafile offset=" + ckoff);
    }
    // create an input stream from the block file
    // and read in partial crc chunk into temporary buffer
    //
    byte[] buf = new byte[sizePartialChunk];
    byte[] crcbuf = new byte[checksumSize];
    try (ReplicaInputStreams instr = datanode.data.getTmpInputStreams(block, blkoff, ckoff)) {
        instr.readDataFully(buf, 0, sizePartialChunk);
        // open meta file and read in crc value computer earlier
        instr.readChecksumFully(crcbuf, 0, crcbuf.length);
    }
    // compute crc of partial chunk from data read in the block file.
    final Checksum partialCrc = DataChecksum.newDataChecksum(diskChecksum.getChecksumType(), diskChecksum.getBytesPerChecksum());
    partialCrc.update(buf, 0, sizePartialChunk);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Read in partial CRC chunk from disk for " + block);
    }
    // recalculated just now
    if (partialCrc.getValue() != checksum2long(crcbuf)) {
        String msg = "Partial CRC " + partialCrc.getValue() + " does not match value computed the " + " last time file was closed " + checksum2long(crcbuf);
        throw new IOException(msg);
    }
    return partialCrc;
}

Example 25 with Checksum

use of java.util.zip.Checksum in project DirectMemory by raffaeleguidi.

the class OffHeapMemoryBuffer method crc32.

public static long crc32(byte[] payload) {
    final Checksum checksum = new CRC32();
    checksum.update(payload, 0, payload.length);
    return checksum.getValue();
}