Examples with FixedWidthType com.facebook.presto.common.type.FixedWidthType used on opensource projects

Example 6 with FixedWidthType

use of com.facebook.presto.common.type.FixedWidthType in project presto by prestodb.

the class AggregatedParquetPageSource method writeMinMax.

private void writeMinMax(ParquetMetadata parquetMetadata, int columnIndex, BlockBuilder blockBuilder, Type type, HiveType hiveType, boolean isMin) {
    org.apache.parquet.schema.Type parquetType = parquetMetadata.getFileMetaData().getSchema().getType(columnIndex);
    if (parquetType instanceof GroupType) {
        throw new IllegalArgumentException("Unsupported type : " + parquetType.toString());
    }
    Object value = null;
    for (BlockMetaData blockMetaData : parquetMetadata.getBlocks()) {
        Statistics statistics = blockMetaData.getColumns().get(columnIndex).getStatistics();
        if (!statistics.hasNonNullValue()) {
            throw new UnsupportedOperationException("No min/max found for parquet file. Set session property hive.pushdown_partial_aggregations_into_scan=false and execute query again");
        }
        if (isMin) {
            Object currentValue = statistics.genericGetMin();
            if (currentValue != null && (value == null || ((Comparable) currentValue).compareTo(value) < 0)) {
                value = currentValue;
            }
        } else {
            Object currentValue = statistics.genericGetMax();
            if (currentValue != null && (value == null || ((Comparable) currentValue).compareTo(value) > 0)) {
                value = currentValue;
            }
        }
    }
    if (type instanceof FixedWidthType) {
        completedBytes += ((FixedWidthType) type).getFixedSize();
    }
    if (value == null) {
        blockBuilder.appendNull();
        return;
    }
    PrimitiveType.PrimitiveTypeName parquetTypeName = parquetType.asPrimitiveType().getPrimitiveTypeName();
    switch(parquetTypeName) {
        case INT32:
            {
                blockBuilder.writeLong(Long.valueOf((Integer) value));
                break;
            }
        case INT64:
            {
                blockBuilder.writeLong((Long) value);
                break;
            }
        case INT96:
            {
                blockBuilder.writeLong(getTimestampMillis(((Binary) value).getBytes(), 0));
                break;
            }
        case FLOAT:
            {
                blockBuilder.writeLong(floatToRawIntBits((Float) value));
                break;
            }
        case DOUBLE:
            {
                type.writeDouble(blockBuilder, (Double) value);
                break;
            }
        case FIXED_LEN_BYTE_ARRAY:
            {
                byte[] valBytes = ((Binary) value).getBytes();
                DecimalType decimalType = (DecimalType) hiveType.getType(typeManager);
                if (decimalType.isShort()) {
                    blockBuilder.writeLong(getShortDecimalValue(valBytes));
                } else {
                    BigInteger bigIntValue = new BigInteger(valBytes);
                    type.writeSlice(blockBuilder, encodeUnscaledValue(bigIntValue));
                }
                break;
            }
        case BINARY:
            {
                Slice slice = Slices.wrappedBuffer(((Binary) value).getBytes());
                blockBuilder.writeBytes(slice, 0, slice.length()).closeEntry();
                completedBytes += slice.length();
                break;
            }
        case BOOLEAN:
        default:
            throw new IllegalArgumentException("Unexpected parquet type name: " + parquetTypeName);
    }
}

Example 7 with FixedWidthType

use of com.facebook.presto.common.type.FixedWidthType in project presto by prestodb.

the class MapFlatSelectiveStreamReader method assembleMapBlock.

private Block assembleMapBlock(boolean includeNulls) {
    offsets = ensureCapacity(offsets, outputPositionCount + 1);
    offsets[0] = 0;
    int offset = 0;
    int inMapIndex = 0;
    for (int i = 0; i < outputPositionCount; i++) {
        if (!includeNulls || !nulls[i]) {
            offset += nestedLengths[inMapIndex];
            inMapIndex++;
        }
        offsets[i + 1] = offset;
    }
    BlockLease[] valueBlockLeases = new BlockLease[keyCount];
    Block[] valueBlocks = new Block[keyCount];
    for (int i = 0; i < keyCount; i++) {
        if (nestedPositionCounts[i] > 0) {
            valueBlockLeases[i] = valueStreamReaders.get(i).getBlockView(nestedOutputPositions[i], nestedPositionCounts[i]);
            valueBlocks[i] = valueBlockLeases[i].get();
        } else {
            valueBlocks[i] = outputType.getKeyType().createBlockBuilder(null, 0).build();
        }
    }
    int[] keyIds = new int[offset];
    int count = 0;
    Type valueType = outputType.getValueType();
    BlockBuilder valueBlockBuilder;
    if (valueType instanceof FixedWidthType) {
        valueBlockBuilder = ((FixedWidthType) valueType).createFixedSizeBlockBuilder(offset);
    } else {
        valueBlockBuilder = valueType.createBlockBuilder(null, offset);
    }
    int[] valueBlockPositions = new int[keyCount];
    inMapIndex = 0;
    for (int i = 0; i < outputPositionCount; i++) {
        if (includeNulls && nulls[i]) {
            continue;
        }
        for (int keyIndex = 0; keyIndex < keyCount; keyIndex++) {
            if (inMap[keyIndex][inMapIndex]) {
                valueType.appendTo(valueBlocks[keyIndex], valueBlockPositions[keyIndex], valueBlockBuilder);
                valueBlockPositions[keyIndex]++;
                keyIds[count++] = keyIndex;
            }
        }
        inMapIndex++;
    }
    for (int i = 0; i < keyCount; i++) {
        if (valueBlockLeases[i] != null) {
            valueBlockLeases[i].close();
        }
    }
    return outputType.createBlockFromKeyValue(outputPositionCount, Optional.ofNullable(includeNulls ? nulls : null), offsets, new DictionaryBlock(keyBlock, keyIds), valueBlockBuilder.build());
}

Example 8 with FixedWidthType

use of com.facebook.presto.common.type.FixedWidthType in project presto by prestodb.

the class TestWriterBlockRawSize method testRowType.

@Test
public void testRowType() {
    Type elementType = INTEGER;
    Type rowType = RowType.anonymous(ImmutableList.of(elementType, elementType));
    ColumnWriter columnWriter = createColumnWriter(rowType);
    BlockBuilder blockBuilder = elementType.createBlockBuilder(null, NUM_ELEMENTS);
    boolean[] isNull = new boolean[NUM_ELEMENTS * 2];
    for (int i = 0; i < NUM_ELEMENTS; i++) {
        elementType.writeLong(blockBuilder, i);
        isNull[i * 2] = true;
    }
    Block elementBlock = blockBuilder.build();
    Block[] elementBlocks = new Block[] { elementBlock, elementBlock };
    Block rowBlock = RowBlock.fromFieldBlocks(NUM_ELEMENTS * 2, Optional.of(isNull), elementBlocks);
    long rawSize = columnWriter.writeBlock(rowBlock);
    long expectedSize = NUM_ELEMENTS + (NUM_ELEMENTS * 2 * ((FixedWidthType) elementType).getFixedSize());
    assertEquals(rawSize, expectedSize);
}

Example 9 with FixedWidthType

use of com.facebook.presto.common.type.FixedWidthType in project presto by prestodb.

the class TestWriterBlockRawSize method testMapType.

@Test
public void testMapType() {
    Type elementType = INTEGER;
    Type arrayType = createMapType(elementType, elementType);
    ColumnWriter columnWriter = createColumnWriter(arrayType);
    BlockBuilder blockBuilder = arrayType.createBlockBuilder(null, NUM_ELEMENTS * 2);
    int totalChildElements = 0;
    for (int i = 0; i < NUM_ELEMENTS; i++) {
        blockBuilder.appendNull();
        BlockBuilder elementBlockBuilder = blockBuilder.beginBlockEntry();
        for (int j = 0; j < i; j++) {
            // key
            elementType.writeLong(elementBlockBuilder, j);
            // value
            elementType.writeLong(elementBlockBuilder, j);
        }
        blockBuilder.closeEntry();
        totalChildElements += i;
    }
    long rawSize = columnWriter.writeBlock(blockBuilder.build());
    long expectedSize = NUM_ELEMENTS + (totalChildElements * 2 * ((FixedWidthType) elementType).getFixedSize());
    assertEquals(rawSize, expectedSize);
}