use of io.prestosql.orc.OrcCorruptionException in project hetu-core by openlookeng.
the class ByteColumnReader method readBlock.
@Override
public Block readBlock() throws IOException {
if (!rowGroupOpen) {
openRowGroup();
}
if (readOffset > 0) {
if (presentStream != null) {
// skip ahead the present bit reader, but count the set bits
// and use this as the skip size for the data reader
readOffset = presentStream.countBitsSet(readOffset);
}
if (readOffset > 0) {
if (dataStream == null) {
throw new OrcCorruptionException(column.getOrcDataSourceId(), "Value is not null but data stream is missing");
}
dataStream.skip(readOffset);
}
}
Block block;
if (dataStream == null) {
if (presentStream == null) {
throw new OrcCorruptionException(column.getOrcDataSourceId(), "Value is null but present stream is missing");
}
presentStream.skip(nextBatchSize);
block = RunLengthEncodedBlock.create(TINYINT, null, nextBatchSize);
} else if (presentStream == null) {
block = readNonNullBlock();
} else {
boolean[] isNull = new boolean[nextBatchSize];
int nullCount = presentStream.getUnsetBits(nextBatchSize, isNull);
if (nullCount == 0) {
block = readNonNullBlock();
} else if (nullCount != nextBatchSize) {
block = readNullBlock(isNull, nextBatchSize - nullCount);
} else {
block = RunLengthEncodedBlock.create(TINYINT, null, nextBatchSize);
}
}
readOffset = 0;
nextBatchSize = 0;
return block;
}
use of io.prestosql.orc.OrcCorruptionException in project hetu-core by openlookeng.
the class DateColumnReader method readBlock.
@Override
public Block<Integer> readBlock() throws IOException {
if (!rowGroupOpen) {
openRowGroup();
}
if (readOffset > 0) {
if (presentStream != null) {
// skip ahead the present bit reader, but count the set bits
// and use this as the skip size for the data reader
readOffset = presentStream.countBitsSet(readOffset);
}
if (readOffset > 0) {
if (dataStream == null) {
throw new OrcCorruptionException(column.getOrcDataSourceId(), "Value is not null but data stream is missing");
}
dataStream.skip(readOffset);
}
}
Block block;
if (dataStream == null) {
if (presentStream == null) {
throw new OrcCorruptionException(column.getOrcDataSourceId(), "Value is null but present stream is missing");
}
presentStream.skip(nextBatchSize);
block = RunLengthEncodedBlock.create(DateType.DATE, null, nextBatchSize);
} else if (presentStream == null) {
block = readNonNullBlock();
} else {
boolean[] isNull = new boolean[nextBatchSize];
int nullCount = presentStream.getUnsetBits(nextBatchSize, isNull);
if (nullCount == 0) {
block = readNonNullBlock();
} else if (nullCount != nextBatchSize) {
block = readNullBlock(isNull, nextBatchSize - nullCount);
} else {
block = RunLengthEncodedBlock.create(DateType.DATE, null, nextBatchSize);
}
}
readOffset = 0;
nextBatchSize = 0;
return block;
}
use of io.prestosql.orc.OrcCorruptionException in project hetu-core by openlookeng.
the class MapColumnReader method readBlock.
@Override
public Block readBlock() throws IOException {
if (!rowGroupOpen) {
openRowGroup();
}
if (readOffset > 0) {
if (presentStream != null) {
// skip ahead the present bit reader, but count the set bits
// and use this as the skip size for the data reader
readOffset = presentStream.countBitsSet(readOffset);
}
if (readOffset > 0) {
if (lengthStream == null) {
throw new OrcCorruptionException(column.getOrcDataSourceId(), "Value is not null but data stream is not present");
}
long entrySkipSize = lengthStream.sum(readOffset);
keyColumnReader.prepareNextRead(toIntExact(entrySkipSize));
valueColumnReader.prepareNextRead(toIntExact(entrySkipSize));
}
}
// We will use the offsetVector as the buffer to read the length values from lengthStream,
// and the length values will be converted in-place to an offset vector.
int[] offsetVector = new int[nextBatchSize + 1];
boolean[] nullVector = null;
if (presentStream == null) {
if (lengthStream == null) {
throw new OrcCorruptionException(column.getOrcDataSourceId(), "Value is not null but data stream is not present");
}
lengthStream.next(offsetVector, nextBatchSize);
} else {
nullVector = new boolean[nextBatchSize];
int nullValues = presentStream.getUnsetBits(nextBatchSize, nullVector);
if (nullValues != nextBatchSize) {
if (lengthStream == null) {
throw new OrcCorruptionException(column.getOrcDataSourceId(), "Value is not null but data stream is not present");
}
lengthStream.next(offsetVector, nextBatchSize - nullValues);
unpackLengthNulls(offsetVector, nullVector, nextBatchSize - nullValues);
}
}
// Calculate the entryCount. Note that the values in the offsetVector are still length values now.
int entryCount = 0;
for (int i = 0; i < offsetVector.length - 1; i++) {
entryCount += offsetVector[i];
}
Block keys;
Block values;
if (entryCount > 0) {
keyColumnReader.prepareNextRead(entryCount);
valueColumnReader.prepareNextRead(entryCount);
keys = keyColumnReader.readBlock();
values = blockFactory.createBlock(entryCount, valueColumnReader::readBlock);
} else {
keys = type.getKeyType().createBlockBuilder(null, 0).build();
values = type.getValueType().createBlockBuilder(null, 1).build();
}
Block[] keyValueBlock = createKeyValueBlock(nextBatchSize, keys, values, offsetVector);
convertLengthVectorToOffsetVector(offsetVector);
readOffset = 0;
nextBatchSize = 0;
return type.createBlockFromKeyValue(Optional.ofNullable(nullVector), offsetVector, keyValueBlock[0], keyValueBlock[1]);
}
use of io.prestosql.orc.OrcCorruptionException in project hetu-core by openlookeng.
the class TimestampColumnReader method readBlock.
@Override
public Block readBlock() throws IOException {
if (!rowGroupOpen) {
openRowGroup();
}
if (readOffset > 0) {
if (presentStream != null) {
// skip ahead the present bit reader, but count the set bits
// and use this as the skip size for the data reader
readOffset = presentStream.countBitsSet(readOffset);
}
if (readOffset > 0) {
if (secondsStream == null) {
throw new OrcCorruptionException(column.getOrcDataSourceId(), "Value is not null but seconds stream is missing");
}
if (nanosStream == null) {
throw new OrcCorruptionException(column.getOrcDataSourceId(), "Value is not null but nanos stream is missing");
}
secondsStream.skip(readOffset);
nanosStream.skip(readOffset);
}
}
Block block;
if (secondsStream == null && nanosStream == null) {
if (presentStream == null) {
throw new OrcCorruptionException(column.getOrcDataSourceId(), "Value is null but present stream is missing");
}
presentStream.skip(nextBatchSize);
block = RunLengthEncodedBlock.create(TIMESTAMP, null, nextBatchSize);
} else if (presentStream == null) {
block = readNonNullBlock();
} else {
boolean[] isNull = new boolean[nextBatchSize];
int nullCount = presentStream.getUnsetBits(nextBatchSize, isNull);
if (nullCount == 0) {
block = readNonNullBlock();
} else if (nullCount != nextBatchSize) {
block = readNullBlock(isNull);
} else {
block = RunLengthEncodedBlock.create(TIMESTAMP, null, nextBatchSize);
}
}
readOffset = 0;
nextBatchSize = 0;
return block;
}
use of io.prestosql.orc.OrcCorruptionException in project hetu-core by openlookeng.
the class ByteInputStream method next.
public void next(byte[] values, int items) throws IOException {
int outputOffset = 0;
while (outputOffset < items) {
if (offset == length) {
readNextBlock();
}
if (length == 0) {
throw new OrcCorruptionException(input.getOrcDataSourceId(), "Unexpected end of stream");
}
int chunkSize = min(items - outputOffset, length - offset);
System.arraycopy(buffer, offset, values, outputOffset, chunkSize);
outputOffset += chunkSize;
offset += chunkSize;
}
}
Aggregations