Examples with FixedWidthType com.facebook.presto.common.type.FixedWidthType used on opensource projects

Example 1 with FixedWidthType

use of com.facebook.presto.common.type.FixedWidthType in project presto by prestodb.

the class TestWriterBlockRawSize method testFileMetadataRawSize.

@Test
public void testFileMetadataRawSize() throws IOException {
    Type type = INTEGER;
    List<Type> types = ImmutableList.of(type);
    int numBlocksPerRowGroup = 3;
    int numBlocksPerStripe = numBlocksPerRowGroup * 5;
    int numStripes = 4;
    int numBlocksPerFile = numBlocksPerStripe * numStripes + 1;
    BlockBuilder blockBuilder = type.createBlockBuilder(null, NUM_ELEMENTS * 2);
    for (int i = 0; i < NUM_ELEMENTS; i++) {
        blockBuilder.appendNull();
        type.writeLong(blockBuilder, i);
    }
    long blockRawSize = ((FixedWidthType) type).getFixedSize() * NUM_ELEMENTS + NUM_ELEMENTS;
    Block block = blockBuilder.build();
    Block[] blocks = new Block[] { block };
    OrcWriterOptions writerOptions = OrcWriterOptions.builder().withRowGroupMaxRowCount(block.getPositionCount() * numBlocksPerRowGroup).withFlushPolicy(DefaultOrcWriterFlushPolicy.builder().withStripeMaxRowCount(block.getPositionCount() * numBlocksPerStripe).build()).build();
    for (OrcEncoding encoding : OrcEncoding.values()) {
        try (TempFile tempFile = new TempFile()) {
            OrcWriter writer = createOrcWriter(tempFile.getFile(), encoding, ZSTD, Optional.empty(), types, writerOptions, new OrcWriterStats());
            for (int i = 0; i < numBlocksPerFile; i++) {
                writer.write(new Page(blocks));
            }
            writer.close();
            writer.validate(new FileOrcDataSource(tempFile.getFile(), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true));
            Footer footer = OrcTester.getFileMetadata(tempFile.getFile(), encoding).getFooter();
            verifyValue(encoding, footer.getRawSize(), blockRawSize * numBlocksPerFile);
            assertEquals(footer.getStripes().size(), numStripes + 1);
            int numBlocksRemaining = numBlocksPerFile;
            for (StripeInformation stripeInfo : footer.getStripes()) {
                int numBlocksInStripe = Math.min(numBlocksRemaining, numBlocksPerStripe);
                verifyValue(encoding, stripeInfo.getRawDataSize(), blockRawSize * numBlocksInStripe);
                numBlocksRemaining -= numBlocksInStripe;
            }
        }
    }
}

Example 2 with FixedWidthType

use of com.facebook.presto.common.type.FixedWidthType in project presto by prestodb.

the class TestWriterBlockRawSize method testArrayType.

@Test
public void testArrayType() {
    Type elementType = INTEGER;
    Type arrayType = new ArrayType(elementType);
    ColumnWriter columnWriter = createColumnWriter(arrayType);
    BlockBuilder blockBuilder = arrayType.createBlockBuilder(null, NUM_ELEMENTS * 2);
    int totalChildElements = 0;
    for (int i = 0; i < NUM_ELEMENTS; i++) {
        blockBuilder.appendNull();
        BlockBuilder elementBlockBuilder = blockBuilder.beginBlockEntry();
        for (int j = 0; j < i; j++) {
            elementType.writeLong(elementBlockBuilder, j);
        }
        blockBuilder.closeEntry();
        totalChildElements += i;
    }
    long rawSize = columnWriter.writeBlock(blockBuilder.build());
    long expectedSize = NUM_ELEMENTS + (totalChildElements * ((FixedWidthType) elementType).getFixedSize());
    assertEquals(rawSize, expectedSize);
}

Example 3 with FixedWidthType

use of com.facebook.presto.common.type.FixedWidthType in project presto by prestodb.

the class AggregatedOrcPageSource method writeMinMax.

private void writeMinMax(int columnIndex, Type type, HiveType hiveType, BlockBuilder blockBuilder, boolean isMin) {
    ColumnStatistics columnStatistics = footer.getFileStats().get(columnIndex + 1);
    OrcType orcType = footer.getTypes().get(columnIndex + 1);
    if (type instanceof FixedWidthType) {
        completedBytes += ((FixedWidthType) type).getFixedSize();
    }
    String orcNoMinMaxMessage = "No min/max found for orc file. Set session property hive.pushdown_partial_aggregations_into_scan=false and execute query again";
    switch(orcType.getOrcTypeKind()) {
        case SHORT:
        case INT:
        case LONG:
            {
                Long value = isMin ? columnStatistics.getIntegerStatistics().getMin() : columnStatistics.getIntegerStatistics().getMax();
                if (value == null) {
                    throw new UnsupportedOperationException(orcNoMinMaxMessage);
                } else {
                    blockBuilder.writeLong(value);
                }
                break;
            }
        case TIMESTAMP:
        case DATE:
            {
                Integer value = isMin ? columnStatistics.getDateStatistics().getMin() : columnStatistics.getDateStatistics().getMax();
                if (value == null) {
                    throw new UnsupportedOperationException(orcNoMinMaxMessage);
                } else {
                    blockBuilder.writeLong(Long.valueOf(value));
                }
                break;
            }
        case VARCHAR:
        case CHAR:
        case STRING:
            {
                Slice value = isMin ? columnStatistics.getStringStatistics().getMin() : columnStatistics.getStringStatistics().getMax();
                if (value == null) {
                    throw new UnsupportedOperationException(orcNoMinMaxMessage);
                } else {
                    blockBuilder.writeBytes(value, 0, value.length()).closeEntry();
                    completedBytes += value.length();
                }
                break;
            }
        case FLOAT:
            {
                Double value = isMin ? columnStatistics.getDoubleStatistics().getMin() : columnStatistics.getDoubleStatistics().getMax();
                if (value == null) {
                    throw new UnsupportedOperationException(orcNoMinMaxMessage);
                } else {
                    blockBuilder.writeLong(floatToRawIntBits(value.floatValue()));
                }
                break;
            }
        case DOUBLE:
            {
                Double value = isMin ? columnStatistics.getDoubleStatistics().getMin() : columnStatistics.getDoubleStatistics().getMax();
                if (value == null) {
                    throw new UnsupportedOperationException(orcNoMinMaxMessage);
                } else {
                    type.writeDouble(blockBuilder, value);
                }
                break;
            }
        case DECIMAL:
            BigDecimal value = isMin ? columnStatistics.getDecimalStatistics().getMin() : columnStatistics.getDecimalStatistics().getMax();
            if (value == null) {
                throw new UnsupportedOperationException(orcNoMinMaxMessage);
            } else {
                Type definedType = hiveType.getType(typeManager);
                if (Decimals.isShortDecimal(definedType)) {
                    blockBuilder.writeLong(value.unscaledValue().longValue());
                } else {
                    type.writeSlice(blockBuilder, Decimals.encodeUnscaledValue(value.unscaledValue()));
                }
            }
            break;
        case BYTE:
        case BOOLEAN:
        case BINARY:
        case UNION:
        case LIST:
        case STRUCT:
        case MAP:
        default:
            throw new IllegalArgumentException("Unsupported type: " + orcType.getOrcTypeKind());
    }
}

Example 4 with FixedWidthType

use of com.facebook.presto.common.type.FixedWidthType in project presto by prestodb.

the class PinotBrokerPageSourceBase method setValue.

protected void setValue(Type type, BlockBuilder blockBuilder, String value) {
    if (blockBuilder == null) {
        return;
    }
    if (value == null) {
        blockBuilder.appendNull();
        return;
    }
    if (!(type instanceof FixedWidthType) && !(type instanceof VarcharType) && !(type instanceof JsonType)) {
        throw new PinotException(PINOT_UNSUPPORTED_COLUMN_TYPE, Optional.empty(), "type '" + type + "' not supported");
    }
    if (type instanceof FixedWidthType) {
        completedBytes += ((FixedWidthType) type).getFixedSize();
        if (type instanceof BigintType) {
            type.writeLong(blockBuilder, parseDouble(value).longValue());
        } else if (type instanceof IntegerType) {
            blockBuilder.writeInt(parseDouble(value).intValue());
        } else if (type instanceof TinyintType) {
            blockBuilder.writeByte(parseDouble(value).byteValue());
        } else if (type instanceof SmallintType) {
            blockBuilder.writeShort(parseDouble(value).shortValue());
        } else if (type instanceof BooleanType) {
            type.writeBoolean(blockBuilder, parseBoolean(value));
        } else if (type instanceof DecimalType || type instanceof DoubleType) {
            type.writeDouble(blockBuilder, parseDouble(value));
        } else if (type instanceof TimestampType) {
            type.writeLong(blockBuilder, parseTimestamp(value));
        } else if (type instanceof DateType) {
            type.writeLong(blockBuilder, parseLong(value));
        } else {
            throw new PinotException(PINOT_UNSUPPORTED_COLUMN_TYPE, Optional.empty(), "type '" + type + "' not supported");
        }
    } else {
        Slice slice = Slices.utf8Slice(value);
        blockBuilder.writeBytes(slice, 0, slice.length()).closeEntry();
        completedBytes += slice.length();
    }
}

Example 5 with FixedWidthType

use of com.facebook.presto.common.type.FixedWidthType in project presto by prestodb.

the class BlackHolePageSourceProvider method createZeroBlock.

private Block createZeroBlock(Type type, int rowsCount, Slice constantSlice) {
    checkArgument(isSupportedType(type), "Unsupported type [%s]", type);
    Slice slice;
    // do not exceed varchar limit
    if (isVarcharType(type)) {
        slice = constantSlice.slice(0, Math.min(((VarcharType) type).getLength(), constantSlice.length()));
    } else if (isLongDecimal(type)) {
        slice = encodeScaledValue(ZERO);
    } else {
        slice = constantSlice;
    }
    BlockBuilder builder;
    if (type instanceof FixedWidthType) {
        builder = type.createBlockBuilder(null, rowsCount);
    } else {
        builder = type.createBlockBuilder(null, rowsCount, slice.length());
    }
    for (int i = 0; i < rowsCount; i++) {
        Class<?> javaType = type.getJavaType();
        if (javaType == boolean.class) {
            type.writeBoolean(builder, false);
        } else if (javaType == long.class) {
            type.writeLong(builder, 0);
        } else if (javaType == double.class) {
            type.writeDouble(builder, 0.0);
        } else if (javaType == Slice.class) {
            requireNonNull(slice, "slice is null");
            type.writeSlice(builder, slice, 0, slice.length());
        } else {
            throw new UnsupportedOperationException("Unknown javaType: " + javaType.getName());
        }
    }
    return builder.build();
}