Examples with LimitInputStream org.apache.hadoop.hbase.io.LimitInputStream used on opensource projects

Example 1 with LimitInputStream

use of org.apache.hadoop.hbase.io.LimitInputStream in project hbase by apache.

the class ProtobufUtil method mergeDelimitedFrom.

/**
   * This version of protobuf's mergeDelimitedFrom avoid the hard-coded 64MB limit for decoding
   * buffers
   * @param builder current message builder
   * @param in Inputsream with delimited protobuf data
   * @throws IOException
   */
public static void mergeDelimitedFrom(Message.Builder builder, InputStream in) throws IOException {
    // This used to be builder.mergeDelimitedFrom(in);
    // but is replaced to allow us to bump the protobuf size limit.
    final int firstByte = in.read();
    if (firstByte != -1) {
        final int size = CodedInputStream.readRawVarint32(firstByte, in);
        final InputStream limitedInput = new LimitInputStream(in, size);
        final CodedInputStream codedInput = CodedInputStream.newInstance(limitedInput);
        codedInput.setSizeLimit(size);
        builder.mergeFrom(codedInput);
        codedInput.checkLastTagWas(0);
    }
}

Example 2 with LimitInputStream

use of org.apache.hadoop.hbase.io.LimitInputStream in project hbase by apache.

the class ProtobufLogReader method readNext.

@Override
protected boolean readNext(Entry entry) throws IOException {
    while (true) {
        // OriginalPosition might be < 0 on local fs; if so, it is useless to us.
        long originalPosition = this.inputStream.getPos();
        if (trailerPresent && originalPosition > 0 && originalPosition == this.walEditsStopOffset) {
            if (LOG.isTraceEnabled()) {
                LOG.trace("Reached end of expected edits area at offset " + originalPosition);
            }
            return false;
        }
        WALKey.Builder builder = WALKey.newBuilder();
        long size = 0;
        try {
            long available = -1;
            try {
                int firstByte = this.inputStream.read();
                if (firstByte == -1) {
                    throw new EOFException("First byte is negative at offset " + originalPosition);
                }
                size = CodedInputStream.readRawVarint32(firstByte, this.inputStream);
                // available may be < 0 on local fs for instance.  If so, can't depend on it.
                available = this.inputStream.available();
                if (available > 0 && available < size) {
                    throw new EOFException("Available stream not enough for edit, " + "inputStream.available()= " + this.inputStream.available() + ", " + "entry size= " + size + " at offset = " + this.inputStream.getPos());
                }
                ProtobufUtil.mergeFrom(builder, new LimitInputStream(this.inputStream, size), (int) size);
            } catch (InvalidProtocolBufferException ipbe) {
                throw (EOFException) new EOFException("Invalid PB, EOF? Ignoring; originalPosition=" + originalPosition + ", currentPosition=" + this.inputStream.getPos() + ", messageSize=" + size + ", currentAvailable=" + available).initCause(ipbe);
            }
            if (!builder.isInitialized()) {
                //       If we can get the KV count, we could, theoretically, try to get next record.
                throw new EOFException("Partial PB while reading WAL, " + "probably an unexpected EOF, ignoring. current offset=" + this.inputStream.getPos());
            }
            WALKey walKey = builder.build();
            entry.getKey().readFieldsFromPb(walKey, this.byteStringUncompressor);
            if (!walKey.hasFollowingKvCount() || 0 == walKey.getFollowingKvCount()) {
                if (LOG.isTraceEnabled()) {
                    LOG.trace("WALKey has no KVs that follow it; trying the next one. current offset=" + this.inputStream.getPos());
                }
                continue;
            }
            int expectedCells = walKey.getFollowingKvCount();
            long posBefore = this.inputStream.getPos();
            try {
                int actualCells = entry.getEdit().readFromCells(cellDecoder, expectedCells);
                if (expectedCells != actualCells) {
                    // other info added in catch
                    throw new EOFException("Only read " + actualCells);
                }
            } catch (Exception ex) {
                String posAfterStr = "<unknown>";
                try {
                    posAfterStr = this.inputStream.getPos() + "";
                } catch (Throwable t) {
                    if (LOG.isTraceEnabled()) {
                        LOG.trace("Error getting pos for error message - ignoring", t);
                    }
                }
                String message = " while reading " + expectedCells + " WAL KVs; started reading at " + posBefore + " and read up to " + posAfterStr;
                IOException realEofEx = extractHiddenEof(ex);
                throw (EOFException) new EOFException("EOF " + message).initCause(realEofEx != null ? realEofEx : ex);
            }
            if (trailerPresent && this.inputStream.getPos() > this.walEditsStopOffset) {
                LOG.error("Read WALTrailer while reading WALEdits. wal: " + this.path + ", inputStream.getPos(): " + this.inputStream.getPos() + ", walEditsStopOffset: " + this.walEditsStopOffset);
                throw new EOFException("Read WALTrailer while reading WALEdits");
            }
        } catch (EOFException eof) {
            // If originalPosition is < 0, it is rubbish and we cannot use it (probably local fs)
            if (originalPosition < 0) {
                if (LOG.isTraceEnabled()) {
                    LOG.trace("Encountered a malformed edit, but can't seek back to last good position because originalPosition is negative. last offset=" + this.inputStream.getPos(), eof);
                }
                throw eof;
            }
            // read successfully.
            if (LOG.isTraceEnabled()) {
                LOG.trace("Encountered a malformed edit, seeking back to last good position in file, from " + inputStream.getPos() + " to " + originalPosition, eof);
            }
            seekOnFs(originalPosition);
            return false;
        }
        return true;
    }
}