use of io.prestosql.spi.block.VariableWidthBlock in project carbondata by apache.
the class SliceStreamReader method setDictionary.
@Override
public void setDictionary(CarbonDictionary dictionary) {
super.setDictionary(dictionary);
if (dictionary == null) {
dictionaryBlock = null;
this.isLocalDict = false;
return;
}
boolean[] nulls = new boolean[dictionary.getDictionarySize()];
nulls[0] = true;
nulls[1] = true;
int[] dictOffsets = new int[dictionary.getDictionarySize() + 1];
int size = 0;
for (int i = 0; i < dictionary.getDictionarySize(); i++) {
dictOffsets[i] = size;
if (dictionary.getDictionaryValue(i) != null) {
size += dictionary.getDictionaryValue(i).length;
}
}
byte[] singleArrayDictValues = new byte[size];
for (int i = 0; i < dictionary.getDictionarySize(); i++) {
if (dictionary.getDictionaryValue(i) != null) {
System.arraycopy(dictionary.getDictionaryValue(i), 0, singleArrayDictValues, dictOffsets[i], dictionary.getDictionaryValue(i).length);
}
}
dictOffsets[dictOffsets.length - 1] = size;
dictionaryBlock = new VariableWidthBlock(dictionary.getDictionarySize(), Slices.wrappedBuffer(singleArrayDictValues), dictOffsets, Optional.of(nulls));
this.isLocalDict = true;
}
use of io.prestosql.spi.block.VariableWidthBlock in project hetu-core by openlookeng.
the class AbstractTestHBaseRowSerializer method testGetMapFromBlock.
/**
* testGetMapFromBlock
*
* @throws IllegalArgumentException
*/
@Test
public void testGetMapFromBlock() throws Exception {
MethodHandle methodHandle = Mockito.mock(MethodHandle.class);
MapType type = new MapType(VARCHAR, BIGINT, methodHandle, methodHandle, methodHandle, methodHandle);
int[] offsets = { 0, 2, 4 };
Block rowkey = new VariableWidthBlock(2, TestSliceUtils.createSlice("12345678901"), offsets, Optional.empty());
HBaseRowSerializer.getMapFromBlock(type, rowkey);
}
use of io.prestosql.spi.block.VariableWidthBlock in project hetu-core by openlookeng.
the class SliceDictionaryColumnReader method setDictionaryBlockData.
private void setDictionaryBlockData(byte[] dictionaryData, int[] dictionaryOffsets, int positionCount) {
verify(positionCount > 0);
// the engine currently uses identity equality to test if dictionaries are the same
if (currentDictionaryData != dictionaryData) {
boolean[] isNull = new boolean[positionCount];
isNull[positionCount - 1] = true;
dictionaryOffsets[positionCount] = dictionaryOffsets[positionCount - 1];
dictionaryBlock = new VariableWidthBlock(positionCount, wrappedBuffer(dictionaryData), dictionaryOffsets, Optional.of(isNull));
currentDictionaryData = dictionaryData;
}
}
use of io.prestosql.spi.block.VariableWidthBlock in project hetu-core by openlookeng.
the class SliceDictionarySelectiveColumnReader method setDictionaryBlockData.
private void setDictionaryBlockData(byte[] dictionaryData, int[] dictionaryOffsets, int positionCount) {
verify(positionCount > 0);
// the engine currently uses identity equality to test if dictionaries are the same
if (currentDictionaryData != dictionaryData) {
boolean[] isNullVector = new boolean[positionCount];
isNullVector[positionCount - 1] = true;
dictionaryOffsets[positionCount] = dictionaryOffsets[positionCount - 1];
dictionaryBlock = new VariableWidthBlock(positionCount, wrappedBuffer(dictionaryData), dictionaryOffsets, Optional.of(isNullVector));
currentDictionaryData = dictionaryData;
}
}
use of io.prestosql.spi.block.VariableWidthBlock in project hetu-core by openlookeng.
the class SliceDirectColumnReader method readBlock.
@Override
public Block readBlock() throws IOException {
if (!rowGroupOpen) {
openRowGroup();
}
if (readOffset > 0) {
if (presentStream != null) {
// skip ahead the present bit reader, but count the set bits
// and use this as the skip size for the length reader
readOffset = presentStream.countBitsSet(readOffset);
}
if (readOffset > 0) {
if (lengthStream == null) {
throw new OrcCorruptionException(column.getOrcDataSourceId(), "Value is not null but length stream is missing");
}
long dataSkipSize = lengthStream.sum(readOffset);
if (dataSkipSize > 0) {
if (dataStream == null) {
throw new OrcCorruptionException(column.getOrcDataSourceId(), "Value is not null but data stream is missing");
}
dataStream.skip(dataSkipSize);
}
}
}
if (lengthStream == null) {
if (presentStream == null) {
throw new OrcCorruptionException(column.getOrcDataSourceId(), "Value is null but present stream is missing");
}
presentStream.skip(nextBatchSize);
Block nullValueBlock = readAllNullsBlock();
readOffset = 0;
nextBatchSize = 0;
return nullValueBlock;
}
// create new isNullVector and offsetVector for VariableWidthBlock
boolean[] isNullVector = null;
// We will use the offsetVector as the buffer to read the length values from lengthStream,
// and the length values will be converted in-place to an offset vector.
int[] offsetVector = new int[nextBatchSize + 1];
if (presentStream == null) {
lengthStream.next(offsetVector, nextBatchSize);
} else {
isNullVector = new boolean[nextBatchSize];
int nullCount = presentStream.getUnsetBits(nextBatchSize, isNullVector);
if (nullCount == nextBatchSize) {
// all nulls
Block nullValueBlock = readAllNullsBlock();
readOffset = 0;
nextBatchSize = 0;
return nullValueBlock;
}
if (lengthStream == null) {
throw new OrcCorruptionException(column.getOrcDataSourceId(), "Value is not null but length stream is missing");
}
if (nullCount == 0) {
isNullVector = null;
lengthStream.next(offsetVector, nextBatchSize);
} else {
lengthStream.next(offsetVector, nextBatchSize - nullCount);
unpackLengthNulls(offsetVector, isNullVector, nextBatchSize - nullCount);
}
}
// Calculate the total length for all entries. Note that the values in the offsetVector are still length values now.
long totalLength = 0;
for (int i = 0; i < nextBatchSize; i++) {
totalLength += offsetVector[i];
}
int currentBatchSize = nextBatchSize;
readOffset = 0;
nextBatchSize = 0;
if (totalLength == 0) {
return new VariableWidthBlock(currentBatchSize, EMPTY_SLICE, offsetVector, Optional.ofNullable(isNullVector));
}
if (totalLength > ONE_GIGABYTE) {
throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Values in column \"%s\" are too large to process for Presto. %s column values are larger than 1GB [%s]", column.getPath(), nextBatchSize, column.getOrcDataSourceId()));
}
if (dataStream == null) {
throw new OrcCorruptionException(column.getOrcDataSourceId(), "Value is not null but data stream is missing");
}
// allocate enough space to read
byte[] data = new byte[toIntExact(totalLength)];
Slice slice = Slices.wrappedBuffer(data);
if (maxCodePointCount < 0) {
// unbounded, simply read all data in on shot
dataStream.next(data, 0, data.length);
convertLengthVectorToOffsetVector(offsetVector);
} else {
// We do the following operations together in the for loop:
// * truncate strings
// * convert original length values in offsetVector into truncated offset values
int currentLength = offsetVector[0];
offsetVector[0] = 0;
for (int i = 1; i <= currentBatchSize; i++) {
int nextLength = offsetVector[i];
if (isNullVector != null && isNullVector[i - 1]) {
checkState(currentLength == 0, "Corruption in slice direct stream: length is non-zero for null entry");
offsetVector[i] = offsetVector[i - 1];
currentLength = nextLength;
continue;
}
int offset = offsetVector[i - 1];
// read data without truncation
dataStream.next(data, offset, offset + currentLength);
// adjust offsetVector with truncated length
int truncatedLength = computeTruncatedLength(slice, offset, currentLength, maxCodePointCount, isCharType);
verify(truncatedLength >= 0);
offsetVector[i] = offset + truncatedLength;
currentLength = nextLength;
}
}
// this can lead to over-retention but unlikely to happen given truncation rarely happens
return new VariableWidthBlock(currentBatchSize, slice, offsetVector, Optional.ofNullable(isNullVector));
}
Aggregations