Examples with OrcCorruptionException io.prestosql.orc.OrcCorruptionException used on opensource projects

Example 21 with OrcCorruptionException

use of io.prestosql.orc.OrcCorruptionException in project hetu-core by openlookeng.

the class CompressedOrcChunkLoader method seekToCheckpoint.

@Override
public void seekToCheckpoint(long checkpoint) throws IOException {
    int compressedOffset = decodeCompressedBlockOffset(checkpoint);
    if (compressedOffset >= dataReader.getSize()) {
        throw new OrcCorruptionException(dataReader.getOrcDataSourceId(), "Seek past end of stream");
    }
    // is the compressed offset within the current compressed buffer
    if (compressedBufferStart <= compressedOffset && compressedOffset < compressedBufferStart + compressedBufferStream.length()) {
        compressedBufferStream.setPosition(compressedOffset - compressedBufferStart);
    } else {
        compressedBufferStart = compressedOffset;
        compressedBufferStream = EMPTY_SLICE.getInput();
    }
    nextUncompressedOffset = decodeDecompressedOffset(checkpoint);
    lastCheckpoint = checkpoint;
}

Example 22 with OrcCorruptionException

use of io.prestosql.orc.OrcCorruptionException in project hetu-core by openlookeng.

the class CompressedOrcChunkLoader method ensureCompressedBytesAvailable.

private void ensureCompressedBytesAvailable(int size) throws IOException {
    // is this within the current buffer?
    if (size <= compressedBufferStream.remaining()) {
        return;
    }
    // is this a read larger than the buffer
    if (size > dataReader.getMaxBufferSize()) {
        throw new OrcCorruptionException(dataReader.getOrcDataSourceId(), "Requested read size (%s bytes) is greater than max buffer size (%s bytes", size, dataReader.getMaxBufferSize());
    }
    // is this a read past the end of the stream
    if (compressedBufferStart + compressedBufferStream.position() + size > dataReader.getSize()) {
        throw new OrcCorruptionException(dataReader.getOrcDataSourceId(), "Read past end of stream");
    }
    compressedBufferStart = compressedBufferStart + toIntExact(compressedBufferStream.position());
    Slice compressedBuffer = dataReader.seekBuffer(compressedBufferStart);
    dataReaderMemoryUsage.setBytes(dataReader.getRetainedSize());
    if (compressedBuffer.length() < size) {
        throw new OrcCorruptionException(dataReader.getOrcDataSourceId(), "Requested read of %s bytes but only %s were bytes", size, compressedBuffer.length());
    }
    compressedBufferStream = compressedBuffer.getInput();
}

Example 23 with OrcCorruptionException

use of io.prestosql.orc.OrcCorruptionException in project hetu-core by openlookeng.

the class DecimalInputStream method decodeShortDecimalTail.

private int decodeShortDecimalTail(long[] result, int inputCount, int batchSize) throws IOException {
    int count = inputCount;
    long low = 0;
    long high = 0;
    long value;
    boolean last = false;
    int offset = 0;
    if (blockOffset == block.length()) {
        advance();
    }
    while (true) {
        value = block.getByte(blockOffset);
        blockOffset++;
        if (offset == 0) {
            low |= (value & 0x7F);
        } else if (offset < 8) {
            low |= (value & 0x7F) << (offset * 7);
        } else if (offset < 11) {
            high |= (value & 0x7F) << ((offset - 8) * 7);
        } else {
            throw new OrcCorruptionException(chunkLoader.getOrcDataSourceId(), "Decimal does not fit long (invalid table schema?)");
        }
        offset++;
        if ((value & 0x80) == 0) {
            if (high > 0xFF) {
                // only 63 - (55) = 8 bits allowed in high
                throw new OrcCorruptionException(chunkLoader.getOrcDataSourceId(), "Decimal does not fit long (invalid table schema?)");
            }
            emitShortDecimal(result, count, low, high);
            count++;
            low = 0;
            high = 0;
            offset = 0;
            if (blockOffset == block.length()) {
                // reset the block and loop around to optimized decoding
                break;
            }
            if (last || count == batchSize) {
                break;
            }
        } else if (blockOffset == block.length()) {
            last = true;
            advance();
        }
    }
    return count;
}

Example 24 with OrcCorruptionException

use of io.prestosql.orc.OrcCorruptionException in project hetu-core by openlookeng.

the class DecimalInputStream method nextLongDecimal.

// result must have at least batchSize * 2 capacity
@SuppressWarnings("PointlessBitwiseExpression")
public void nextLongDecimal(long[] result, int batchSize) throws IOException {
    verify(result.length >= batchSize * 2);
    int count = 0;
    while (count < batchSize) {
        if (blockOffset == block.length()) {
            advance();
        }
        while (blockOffset <= block.length() - 20) {
            // we'll read 2 longs + 1 int
            long low;
            long middle = 0;
            int high = 0;
            // low bits
            long current = block.getLong(blockOffset);
            int zeros = Long.numberOfTrailingZeros(~current & LONG_MASK);
            int end = (zeros + 1) / 8;
            blockOffset += end;
            boolean negative = (current & 1) == 1;
            low = (current & 0x7F_00_00_00_00_00_00_00L) >>> 7;
            low |= (current & 0x7F_00_00_00_00_00_00L) >>> 6;
            low |= (current & 0x7F_00_00_00_00_00L) >>> 5;
            low |= (current & 0x7F_00_00_00_00L) >>> 4;
            low |= (current & 0x7F_00_00_00) >>> 3;
            low |= (current & 0x7F_00_00) >>> 2;
            low |= (current & 0x7F_00) >>> 1;
            low |= (current & 0x7F) >>> 0;
            low = low & ((1L << (end * 7)) - 1);
            // middle bits
            if (zeros == 64) {
                current = block.getLong(blockOffset);
                zeros = Long.numberOfTrailingZeros(~current & LONG_MASK);
                end = (zeros + 1) / 8;
                blockOffset += end;
                middle = (current & 0x7F_00_00_00_00_00_00_00L) >>> 7;
                middle |= (current & 0x7F_00_00_00_00_00_00L) >>> 6;
                middle |= (current & 0x7F_00_00_00_00_00L) >>> 5;
                middle |= (current & 0x7F_00_00_00_00L) >>> 4;
                middle |= (current & 0x7F_00_00_00) >>> 3;
                middle |= (current & 0x7F_00_00) >>> 2;
                middle |= (current & 0x7F_00) >>> 1;
                middle |= (current & 0x7F) >>> 0;
                middle = middle & ((1L << (end * 7)) - 1);
                // high bits
                if (zeros == 64) {
                    int last = block.getInt(blockOffset);
                    zeros = Integer.numberOfTrailingZeros(~last & INT_MASK);
                    end = (zeros + 1) / 8;
                    blockOffset += end;
                    high = (last & 0x7F_00_00) >>> 2;
                    high |= (last & 0x7F_00) >>> 1;
                    high |= (last & 0x7F) >>> 0;
                    high = high & ((1 << (end * 7)) - 1);
                    if (end == 4 || high > 0xFF_FF) {
                        // only 127 - (55 + 56) = 16 bits allowed in high
                        throw new OrcCorruptionException(chunkLoader.getOrcDataSourceId(), "Decimal exceeds 128 bits");
                    }
                }
            }
            emitLongDecimal(result, count, low, middle, high, negative);
            count++;
            if (count == batchSize) {
                return;
            }
        }
        // handle the tail of the current block
        count = decodeLongDecimalTail(result, count, batchSize);
    }
}

Example 25 with OrcCorruptionException

use of io.prestosql.orc.OrcCorruptionException in project hetu-core by openlookeng.

the class DecimalInputStream method decodeLongDecimalTail.

private int decodeLongDecimalTail(long[] result, int inputCount, int batchSize) throws IOException {
    int count = inputCount;
    boolean negative = false;
    long low = 0;
    long middle = 0;
    int high = 0;
    long value;
    boolean last = false;
    if (blockOffset == block.length()) {
        advance();
    }
    int offset = 0;
    while (true) {
        value = block.getByte(blockOffset);
        blockOffset++;
        if (offset == 0) {
            negative = (value & 1) == 1;
            low |= (value & 0x7F);
        } else if (offset < 8) {
            low |= (value & 0x7F) << (offset * 7);
        } else if (offset < 16) {
            middle |= (value & 0x7F) << ((offset - 8) * 7);
        } else if (offset < 19) {
            high |= (value & 0x7F) << ((offset - 16) * 7);
        } else {
            throw new OrcCorruptionException(chunkLoader.getOrcDataSourceId(), "Decimal exceeds 128 bits");
        }
        offset++;
        if ((value & 0x80) == 0) {
            if (high > 0xFF_FF) {
                // only 127 - (55 + 56) = 16 bits allowed in high
                throw new OrcCorruptionException(chunkLoader.getOrcDataSourceId(), "Decimal exceeds 128 bits");
            }
            emitLongDecimal(result, count, low, middle, high, negative);
            count++;
            low = 0;
            middle = 0;
            high = 0;
            offset = 0;
            if (blockOffset == block.length()) {
                // reset the block and loop around to optimized decoding
                break;
            }
            if (last || count == batchSize) {
                break;
            }
        } else if (blockOffset == block.length()) {
            last = true;
            advance();
        }
    }
    return count;
}