Search in sources :

Example 1 with FixedWidthType

use of com.facebook.presto.common.type.FixedWidthType in project presto by prestodb.

the class TestWriterBlockRawSize method testFileMetadataRawSize.

@Test
public void testFileMetadataRawSize() throws IOException {
    Type type = INTEGER;
    List<Type> types = ImmutableList.of(type);
    int numBlocksPerRowGroup = 3;
    int numBlocksPerStripe = numBlocksPerRowGroup * 5;
    int numStripes = 4;
    int numBlocksPerFile = numBlocksPerStripe * numStripes + 1;
    BlockBuilder blockBuilder = type.createBlockBuilder(null, NUM_ELEMENTS * 2);
    for (int i = 0; i < NUM_ELEMENTS; i++) {
        blockBuilder.appendNull();
        type.writeLong(blockBuilder, i);
    }
    long blockRawSize = ((FixedWidthType) type).getFixedSize() * NUM_ELEMENTS + NUM_ELEMENTS;
    Block block = blockBuilder.build();
    Block[] blocks = new Block[] { block };
    OrcWriterOptions writerOptions = OrcWriterOptions.builder().withRowGroupMaxRowCount(block.getPositionCount() * numBlocksPerRowGroup).withFlushPolicy(DefaultOrcWriterFlushPolicy.builder().withStripeMaxRowCount(block.getPositionCount() * numBlocksPerStripe).build()).build();
    for (OrcEncoding encoding : OrcEncoding.values()) {
        try (TempFile tempFile = new TempFile()) {
            OrcWriter writer = createOrcWriter(tempFile.getFile(), encoding, ZSTD, Optional.empty(), types, writerOptions, new OrcWriterStats());
            for (int i = 0; i < numBlocksPerFile; i++) {
                writer.write(new Page(blocks));
            }
            writer.close();
            writer.validate(new FileOrcDataSource(tempFile.getFile(), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true));
            Footer footer = OrcTester.getFileMetadata(tempFile.getFile(), encoding).getFooter();
            verifyValue(encoding, footer.getRawSize(), blockRawSize * numBlocksPerFile);
            assertEquals(footer.getStripes().size(), numStripes + 1);
            int numBlocksRemaining = numBlocksPerFile;
            for (StripeInformation stripeInfo : footer.getStripes()) {
                int numBlocksInStripe = Math.min(numBlocksRemaining, numBlocksPerStripe);
                verifyValue(encoding, stripeInfo.getRawDataSize(), blockRawSize * numBlocksInStripe);
                numBlocksRemaining -= numBlocksInStripe;
            }
        }
    }
}
Also used : OrcWriterStats(com.facebook.presto.orc.OrcWriterStats) OrcWriter(com.facebook.presto.orc.OrcWriter) OrcTester.createOrcWriter(com.facebook.presto.orc.OrcTester.createOrcWriter) Page(com.facebook.presto.common.Page) OrcEncoding(com.facebook.presto.orc.OrcEncoding) OrcWriterOptions(com.facebook.presto.orc.OrcWriterOptions) TestOrcMapNullKey.createMapType(com.facebook.presto.orc.TestOrcMapNullKey.createMapType) TimestampType(com.facebook.presto.common.type.TimestampType) ArrayType(com.facebook.presto.common.type.ArrayType) OrcType(com.facebook.presto.orc.metadata.OrcType) Type(com.facebook.presto.common.type.Type) FixedWidthType(com.facebook.presto.common.type.FixedWidthType) RowType(com.facebook.presto.common.type.RowType) TempFile(com.facebook.presto.orc.TempFile) FileOrcDataSource(com.facebook.presto.orc.FileOrcDataSource) DataSize(io.airlift.units.DataSize) Footer(com.facebook.presto.orc.metadata.Footer) RowBlock(com.facebook.presto.common.block.RowBlock) Block(com.facebook.presto.common.block.Block) StripeInformation(com.facebook.presto.orc.metadata.StripeInformation) BlockBuilder(com.facebook.presto.common.block.BlockBuilder) FixedWidthType(com.facebook.presto.common.type.FixedWidthType) Test(org.testng.annotations.Test)

Example 2 with FixedWidthType

use of com.facebook.presto.common.type.FixedWidthType in project presto by prestodb.

the class TestWriterBlockRawSize method testArrayType.

@Test
public void testArrayType() {
    Type elementType = INTEGER;
    Type arrayType = new ArrayType(elementType);
    ColumnWriter columnWriter = createColumnWriter(arrayType);
    BlockBuilder blockBuilder = arrayType.createBlockBuilder(null, NUM_ELEMENTS * 2);
    int totalChildElements = 0;
    for (int i = 0; i < NUM_ELEMENTS; i++) {
        blockBuilder.appendNull();
        BlockBuilder elementBlockBuilder = blockBuilder.beginBlockEntry();
        for (int j = 0; j < i; j++) {
            elementType.writeLong(elementBlockBuilder, j);
        }
        blockBuilder.closeEntry();
        totalChildElements += i;
    }
    long rawSize = columnWriter.writeBlock(blockBuilder.build());
    long expectedSize = NUM_ELEMENTS + (totalChildElements * ((FixedWidthType) elementType).getFixedSize());
    assertEquals(rawSize, expectedSize);
}
Also used : ArrayType(com.facebook.presto.common.type.ArrayType) TestOrcMapNullKey.createMapType(com.facebook.presto.orc.TestOrcMapNullKey.createMapType) TimestampType(com.facebook.presto.common.type.TimestampType) ArrayType(com.facebook.presto.common.type.ArrayType) OrcType(com.facebook.presto.orc.metadata.OrcType) Type(com.facebook.presto.common.type.Type) FixedWidthType(com.facebook.presto.common.type.FixedWidthType) RowType(com.facebook.presto.common.type.RowType) BlockBuilder(com.facebook.presto.common.block.BlockBuilder) Test(org.testng.annotations.Test)

Example 3 with FixedWidthType

use of com.facebook.presto.common.type.FixedWidthType in project presto by prestodb.

the class AggregatedOrcPageSource method writeMinMax.

private void writeMinMax(int columnIndex, Type type, HiveType hiveType, BlockBuilder blockBuilder, boolean isMin) {
    ColumnStatistics columnStatistics = footer.getFileStats().get(columnIndex + 1);
    OrcType orcType = footer.getTypes().get(columnIndex + 1);
    if (type instanceof FixedWidthType) {
        completedBytes += ((FixedWidthType) type).getFixedSize();
    }
    String orcNoMinMaxMessage = "No min/max found for orc file. Set session property hive.pushdown_partial_aggregations_into_scan=false and execute query again";
    switch(orcType.getOrcTypeKind()) {
        case SHORT:
        case INT:
        case LONG:
            {
                Long value = isMin ? columnStatistics.getIntegerStatistics().getMin() : columnStatistics.getIntegerStatistics().getMax();
                if (value == null) {
                    throw new UnsupportedOperationException(orcNoMinMaxMessage);
                } else {
                    blockBuilder.writeLong(value);
                }
                break;
            }
        case TIMESTAMP:
        case DATE:
            {
                Integer value = isMin ? columnStatistics.getDateStatistics().getMin() : columnStatistics.getDateStatistics().getMax();
                if (value == null) {
                    throw new UnsupportedOperationException(orcNoMinMaxMessage);
                } else {
                    blockBuilder.writeLong(Long.valueOf(value));
                }
                break;
            }
        case VARCHAR:
        case CHAR:
        case STRING:
            {
                Slice value = isMin ? columnStatistics.getStringStatistics().getMin() : columnStatistics.getStringStatistics().getMax();
                if (value == null) {
                    throw new UnsupportedOperationException(orcNoMinMaxMessage);
                } else {
                    blockBuilder.writeBytes(value, 0, value.length()).closeEntry();
                    completedBytes += value.length();
                }
                break;
            }
        case FLOAT:
            {
                Double value = isMin ? columnStatistics.getDoubleStatistics().getMin() : columnStatistics.getDoubleStatistics().getMax();
                if (value == null) {
                    throw new UnsupportedOperationException(orcNoMinMaxMessage);
                } else {
                    blockBuilder.writeLong(floatToRawIntBits(value.floatValue()));
                }
                break;
            }
        case DOUBLE:
            {
                Double value = isMin ? columnStatistics.getDoubleStatistics().getMin() : columnStatistics.getDoubleStatistics().getMax();
                if (value == null) {
                    throw new UnsupportedOperationException(orcNoMinMaxMessage);
                } else {
                    type.writeDouble(blockBuilder, value);
                }
                break;
            }
        case DECIMAL:
            BigDecimal value = isMin ? columnStatistics.getDecimalStatistics().getMin() : columnStatistics.getDecimalStatistics().getMax();
            if (value == null) {
                throw new UnsupportedOperationException(orcNoMinMaxMessage);
            } else {
                Type definedType = hiveType.getType(typeManager);
                if (Decimals.isShortDecimal(definedType)) {
                    blockBuilder.writeLong(value.unscaledValue().longValue());
                } else {
                    type.writeSlice(blockBuilder, Decimals.encodeUnscaledValue(value.unscaledValue()));
                }
            }
            break;
        case BYTE:
        case BOOLEAN:
        case BINARY:
        case UNION:
        case LIST:
        case STRUCT:
        case MAP:
        default:
            throw new IllegalArgumentException("Unsupported type: " + orcType.getOrcTypeKind());
    }
}
Also used : ColumnStatistics(com.facebook.presto.orc.metadata.statistics.ColumnStatistics) HiveType(com.facebook.presto.hive.HiveType) FixedWidthType(com.facebook.presto.common.type.FixedWidthType) OrcType(com.facebook.presto.orc.metadata.OrcType) Type(com.facebook.presto.common.type.Type) OrcType(com.facebook.presto.orc.metadata.OrcType) Slice(io.airlift.slice.Slice) BigDecimal(java.math.BigDecimal) FixedWidthType(com.facebook.presto.common.type.FixedWidthType)

Example 4 with FixedWidthType

use of com.facebook.presto.common.type.FixedWidthType in project presto by prestodb.

the class PinotBrokerPageSourceBase method setValue.

protected void setValue(Type type, BlockBuilder blockBuilder, String value) {
    if (blockBuilder == null) {
        return;
    }
    if (value == null) {
        blockBuilder.appendNull();
        return;
    }
    if (!(type instanceof FixedWidthType) && !(type instanceof VarcharType) && !(type instanceof JsonType)) {
        throw new PinotException(PINOT_UNSUPPORTED_COLUMN_TYPE, Optional.empty(), "type '" + type + "' not supported");
    }
    if (type instanceof FixedWidthType) {
        completedBytes += ((FixedWidthType) type).getFixedSize();
        if (type instanceof BigintType) {
            type.writeLong(blockBuilder, parseDouble(value).longValue());
        } else if (type instanceof IntegerType) {
            blockBuilder.writeInt(parseDouble(value).intValue());
        } else if (type instanceof TinyintType) {
            blockBuilder.writeByte(parseDouble(value).byteValue());
        } else if (type instanceof SmallintType) {
            blockBuilder.writeShort(parseDouble(value).shortValue());
        } else if (type instanceof BooleanType) {
            type.writeBoolean(blockBuilder, parseBoolean(value));
        } else if (type instanceof DecimalType || type instanceof DoubleType) {
            type.writeDouble(blockBuilder, parseDouble(value));
        } else if (type instanceof TimestampType) {
            type.writeLong(blockBuilder, parseTimestamp(value));
        } else if (type instanceof DateType) {
            type.writeLong(blockBuilder, parseLong(value));
        } else {
            throw new PinotException(PINOT_UNSUPPORTED_COLUMN_TYPE, Optional.empty(), "type '" + type + "' not supported");
        }
    } else {
        Slice slice = Slices.utf8Slice(value);
        blockBuilder.writeBytes(slice, 0, slice.length()).closeEntry();
        completedBytes += slice.length();
    }
}
Also used : JsonType(com.facebook.presto.common.type.JsonType) VarcharType(com.facebook.presto.common.type.VarcharType) TinyintType(com.facebook.presto.common.type.TinyintType) BooleanType(com.facebook.presto.common.type.BooleanType) BigintType(com.facebook.presto.common.type.BigintType) IntegerType(com.facebook.presto.common.type.IntegerType) DoubleType(com.facebook.presto.common.type.DoubleType) Slice(io.airlift.slice.Slice) DecimalType(com.facebook.presto.common.type.DecimalType) TimestampType(com.facebook.presto.common.type.TimestampType) SmallintType(com.facebook.presto.common.type.SmallintType) DateType(com.facebook.presto.common.type.DateType) FixedWidthType(com.facebook.presto.common.type.FixedWidthType)

Example 5 with FixedWidthType

use of com.facebook.presto.common.type.FixedWidthType in project presto by prestodb.

the class BlackHolePageSourceProvider method createZeroBlock.

private Block createZeroBlock(Type type, int rowsCount, Slice constantSlice) {
    checkArgument(isSupportedType(type), "Unsupported type [%s]", type);
    Slice slice;
    // do not exceed varchar limit
    if (isVarcharType(type)) {
        slice = constantSlice.slice(0, Math.min(((VarcharType) type).getLength(), constantSlice.length()));
    } else if (isLongDecimal(type)) {
        slice = encodeScaledValue(ZERO);
    } else {
        slice = constantSlice;
    }
    BlockBuilder builder;
    if (type instanceof FixedWidthType) {
        builder = type.createBlockBuilder(null, rowsCount);
    } else {
        builder = type.createBlockBuilder(null, rowsCount, slice.length());
    }
    for (int i = 0; i < rowsCount; i++) {
        Class<?> javaType = type.getJavaType();
        if (javaType == boolean.class) {
            type.writeBoolean(builder, false);
        } else if (javaType == long.class) {
            type.writeLong(builder, 0);
        } else if (javaType == double.class) {
            type.writeDouble(builder, 0.0);
        } else if (javaType == Slice.class) {
            requireNonNull(slice, "slice is null");
            type.writeSlice(builder, slice, 0, slice.length());
        } else {
            throw new UnsupportedOperationException("Unknown javaType: " + javaType.getName());
        }
    }
    return builder.build();
}
Also used : Slice(io.airlift.slice.Slice) BlockBuilder(com.facebook.presto.common.block.BlockBuilder) FixedWidthType(com.facebook.presto.common.type.FixedWidthType)

Aggregations

FixedWidthType (com.facebook.presto.common.type.FixedWidthType)9 BlockBuilder (com.facebook.presto.common.block.BlockBuilder)6 Type (com.facebook.presto.common.type.Type)6 TimestampType (com.facebook.presto.common.type.TimestampType)5 OrcType (com.facebook.presto.orc.metadata.OrcType)5 ArrayType (com.facebook.presto.common.type.ArrayType)4 RowType (com.facebook.presto.common.type.RowType)4 TestOrcMapNullKey.createMapType (com.facebook.presto.orc.TestOrcMapNullKey.createMapType)4 Slice (io.airlift.slice.Slice)4 Test (org.testng.annotations.Test)4 Block (com.facebook.presto.common.block.Block)3 RowBlock (com.facebook.presto.common.block.RowBlock)2 BigintType (com.facebook.presto.common.type.BigintType)2 DecimalType (com.facebook.presto.common.type.DecimalType)2 IntegerType (com.facebook.presto.common.type.IntegerType)2 SmallintType (com.facebook.presto.common.type.SmallintType)2 TinyintType (com.facebook.presto.common.type.TinyintType)2 Page (com.facebook.presto.common.Page)1 BlockLease (com.facebook.presto.common.block.BlockLease)1 ClosingBlockLease (com.facebook.presto.common.block.ClosingBlockLease)1