Search in sources :

Example 1 with SqlDecimal

use of com.facebook.presto.common.type.SqlDecimal in project presto by prestodb.

the class OrcTester method decodeRecordReaderValue.

private static Object decodeRecordReaderValue(Type type, Object actualValue) {
    if (actualValue instanceof OrcLazyObject) {
        try {
            actualValue = ((OrcLazyObject) actualValue).materialize();
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }
    }
    if (actualValue instanceof BooleanWritable) {
        actualValue = ((BooleanWritable) actualValue).get();
    } else if (actualValue instanceof ByteWritable) {
        actualValue = ((ByteWritable) actualValue).get();
    } else if (actualValue instanceof BytesWritable) {
        actualValue = new SqlVarbinary(((BytesWritable) actualValue).copyBytes());
    } else if (actualValue instanceof DateWritable) {
        actualValue = new SqlDate(((DateWritable) actualValue).getDays());
    } else if (actualValue instanceof DoubleWritable) {
        actualValue = ((DoubleWritable) actualValue).get();
    } else if (actualValue instanceof FloatWritable) {
        actualValue = ((FloatWritable) actualValue).get();
    } else if (actualValue instanceof IntWritable) {
        actualValue = ((IntWritable) actualValue).get();
    } else if (actualValue instanceof HiveCharWritable) {
        actualValue = ((HiveCharWritable) actualValue).getPaddedValue().toString();
    } else if (actualValue instanceof LongWritable) {
        actualValue = ((LongWritable) actualValue).get();
    } else if (actualValue instanceof ShortWritable) {
        actualValue = ((ShortWritable) actualValue).get();
    } else if (actualValue instanceof HiveDecimalWritable) {
        DecimalType decimalType = (DecimalType) type;
        HiveDecimalWritable writable = (HiveDecimalWritable) actualValue;
        // writable messes with the scale so rescale the values to the Presto type
        BigInteger rescaledValue = rescale(writable.getHiveDecimal().unscaledValue(), writable.getScale(), decimalType.getScale());
        actualValue = new SqlDecimal(rescaledValue, decimalType.getPrecision(), decimalType.getScale());
    } else if (actualValue instanceof Text) {
        actualValue = actualValue.toString();
    } else if (actualValue instanceof TimestampWritable) {
        TimestampWritable timestamp = (TimestampWritable) actualValue;
        actualValue = sqlTimestampOf((timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L), SESSION);
    } else if (actualValue instanceof OrcStruct) {
        List<Object> fields = new ArrayList<>();
        OrcStruct structObject = (OrcStruct) actualValue;
        for (int fieldId = 0; fieldId < structObject.getNumFields(); fieldId++) {
            fields.add(OrcUtil.getFieldValue(structObject, fieldId));
        }
        actualValue = decodeRecordReaderStruct(type, fields);
    } else if (actualValue instanceof com.facebook.hive.orc.OrcStruct) {
        List<Object> fields = new ArrayList<>();
        com.facebook.hive.orc.OrcStruct structObject = (com.facebook.hive.orc.OrcStruct) actualValue;
        for (int fieldId = 0; fieldId < structObject.getNumFields(); fieldId++) {
            fields.add(structObject.getFieldValue(fieldId));
        }
        actualValue = decodeRecordReaderStruct(type, fields);
    } else if (actualValue instanceof List) {
        actualValue = decodeRecordReaderList(type, ((List<?>) actualValue));
    } else if (actualValue instanceof Map) {
        actualValue = decodeRecordReaderMap(type, (Map<?, ?>) actualValue);
    }
    return actualValue;
}
Also used : SqlVarbinary(com.facebook.presto.common.type.SqlVarbinary) Lists.newArrayList(com.google.common.collect.Lists.newArrayList) ArrayList(java.util.ArrayList) TimestampWritable(org.apache.hadoop.hive.serde2.io.TimestampWritable) UncheckedIOException(java.io.UncheckedIOException) DoubleWritable(org.apache.hadoop.io.DoubleWritable) ShortWritable(org.apache.hadoop.hive.serde2.io.ShortWritable) OrcLazyObject(com.facebook.hive.orc.lazy.OrcLazyObject) OrcStruct(org.apache.hadoop.hive.ql.io.orc.OrcStruct) Arrays.asList(java.util.Arrays.asList) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Lists.newArrayList(com.google.common.collect.Lists.newArrayList) ArrayList(java.util.ArrayList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) Collectors.toList(java.util.stream.Collectors.toList) LongWritable(org.apache.hadoop.io.LongWritable) ByteWritable(org.apache.hadoop.io.ByteWritable) IntWritable(org.apache.hadoop.io.IntWritable) DateWritable(org.apache.hadoop.hive.serde2.io.DateWritable) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) HiveCharWritable(org.apache.hadoop.hive.serde2.io.HiveCharWritable) BytesWritable(org.apache.hadoop.io.BytesWritable) SqlDecimal(com.facebook.presto.common.type.SqlDecimal) Text(org.apache.hadoop.io.Text) UncheckedIOException(java.io.UncheckedIOException) IOException(java.io.IOException) FloatWritable(org.apache.hadoop.io.FloatWritable) BooleanWritable(org.apache.hadoop.io.BooleanWritable) SqlDate(com.facebook.presto.common.type.SqlDate) DecimalType(com.facebook.presto.common.type.DecimalType) BigInteger(java.math.BigInteger) OrcLazyObject(com.facebook.hive.orc.lazy.OrcLazyObject) Map(java.util.Map) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap)

Example 2 with SqlDecimal

use of com.facebook.presto.common.type.SqlDecimal in project presto by prestodb.

the class ParquetTester method writeValue.

private static void writeValue(Type type, BlockBuilder blockBuilder, Object value) {
    if (value == null) {
        blockBuilder.appendNull();
    } else {
        if (BOOLEAN.equals(type)) {
            type.writeBoolean(blockBuilder, (Boolean) value);
        } else if (TINYINT.equals(type) || SMALLINT.equals(type) || INTEGER.equals(type) || BIGINT.equals(type)) {
            type.writeLong(blockBuilder, ((Number) value).longValue());
        } else if (Decimals.isShortDecimal(type)) {
            type.writeLong(blockBuilder, ((SqlDecimal) value).getUnscaledValue().longValue());
        } else if (Decimals.isLongDecimal(type)) {
            if (Decimals.overflows(((SqlDecimal) value).getUnscaledValue(), MAX_PRECISION_INT64)) {
                type.writeSlice(blockBuilder, Decimals.encodeUnscaledValue(((SqlDecimal) value).toBigDecimal().unscaledValue()));
            } else {
                type.writeSlice(blockBuilder, Decimals.encodeUnscaledValue(((SqlDecimal) value).getUnscaledValue().longValue()));
            }
        } else if (DOUBLE.equals(type)) {
            type.writeDouble(blockBuilder, ((Number) value).doubleValue());
        } else if (REAL.equals(type)) {
            float floatValue = ((Number) value).floatValue();
            type.writeLong(blockBuilder, Float.floatToIntBits(floatValue));
        } else if (type instanceof VarcharType) {
            Slice slice = truncateToLength(utf8Slice((String) value), type);
            type.writeSlice(blockBuilder, slice);
        } else if (type instanceof CharType) {
            Slice slice = truncateToLengthAndTrimSpaces(utf8Slice((String) value), type);
            type.writeSlice(blockBuilder, slice);
        } else if (VARBINARY.equals(type)) {
            type.writeSlice(blockBuilder, Slices.wrappedBuffer(((SqlVarbinary) value).getBytes()));
        } else if (DATE.equals(type)) {
            long days = ((SqlDate) value).getDays();
            type.writeLong(blockBuilder, days);
        } else if (TIMESTAMP.equals(type)) {
            long millis = ((SqlTimestamp) value).getMillisUtc();
            type.writeLong(blockBuilder, millis);
        } else {
            if (type instanceof ArrayType) {
                List<?> array = (List<?>) value;
                Type elementType = type.getTypeParameters().get(0);
                BlockBuilder arrayBlockBuilder = blockBuilder.beginBlockEntry();
                for (Object elementValue : array) {
                    writeValue(elementType, arrayBlockBuilder, elementValue);
                }
                blockBuilder.closeEntry();
            } else if (type instanceof MapType) {
                Map<?, ?> map = (Map<?, ?>) value;
                Type keyType = type.getTypeParameters().get(0);
                Type valueType = type.getTypeParameters().get(1);
                BlockBuilder mapBlockBuilder = blockBuilder.beginBlockEntry();
                for (Map.Entry<?, ?> entry : map.entrySet()) {
                    writeValue(keyType, mapBlockBuilder, entry.getKey());
                    writeValue(valueType, mapBlockBuilder, entry.getValue());
                }
                blockBuilder.closeEntry();
            } else if (type instanceof RowType) {
                List<?> array = (List<?>) value;
                List<Type> fieldTypes = type.getTypeParameters();
                BlockBuilder rowBlockBuilder = blockBuilder.beginBlockEntry();
                for (int fieldId = 0; fieldId < fieldTypes.size(); fieldId++) {
                    Type fieldType = fieldTypes.get(fieldId);
                    writeValue(fieldType, rowBlockBuilder, array.get(fieldId));
                }
                blockBuilder.closeEntry();
            } else {
                throw new IllegalArgumentException("Unsupported type " + type);
            }
        }
    }
}
Also used : Varchars.isVarcharType(com.facebook.presto.common.type.Varchars.isVarcharType) VarcharType(com.facebook.presto.common.type.VarcharType) SqlVarbinary(com.facebook.presto.common.type.SqlVarbinary) RowType(com.facebook.presto.common.type.RowType) MetastoreUtil.isRowType(com.facebook.presto.hive.metastore.MetastoreUtil.isRowType) SqlDecimal(com.facebook.presto.common.type.SqlDecimal) SqlTimestamp(com.facebook.presto.common.type.SqlTimestamp) MapType(com.facebook.presto.common.type.MapType) MetastoreUtil.isMapType(com.facebook.presto.hive.metastore.MetastoreUtil.isMapType) ArrayType(com.facebook.presto.common.type.ArrayType) MetastoreUtil.isArrayType(com.facebook.presto.hive.metastore.MetastoreUtil.isArrayType) Varchars.isVarcharType(com.facebook.presto.common.type.Varchars.isVarcharType) DecimalType(com.facebook.presto.common.type.DecimalType) HiveUtil.isStructuralType(com.facebook.presto.hive.HiveUtil.isStructuralType) ArrayType(com.facebook.presto.common.type.ArrayType) CharType(com.facebook.presto.common.type.CharType) MetastoreUtil.isArrayType(com.facebook.presto.hive.metastore.MetastoreUtil.isArrayType) RowType(com.facebook.presto.common.type.RowType) MetastoreUtil.isRowType(com.facebook.presto.hive.metastore.MetastoreUtil.isRowType) VarcharType(com.facebook.presto.common.type.VarcharType) MessageType(org.apache.parquet.schema.MessageType) MapType(com.facebook.presto.common.type.MapType) Type(com.facebook.presto.common.type.Type) MetastoreUtil.isMapType(com.facebook.presto.hive.metastore.MetastoreUtil.isMapType) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) Slice(io.airlift.slice.Slice) Collections.singletonList(java.util.Collections.singletonList) ArrayList(java.util.ArrayList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) CharType(com.facebook.presto.common.type.CharType) Map(java.util.Map) HashMap(java.util.HashMap) BlockBuilder(com.facebook.presto.common.block.BlockBuilder)

Example 3 with SqlDecimal

use of com.facebook.presto.common.type.SqlDecimal in project presto by prestodb.

the class AbstractTestParquetReader method testDecimalBackedByINT64.

@Test
public void testDecimalBackedByINT64() throws Exception {
    for (int precision = MAX_PRECISION_INT32 + 1; precision <= MAX_PRECISION_INT64; precision++) {
        int scale = ThreadLocalRandom.current().nextInt(precision);
        MessageType parquetSchema = parseMessageType(format("message hive_decimal { optional INT64 test (DECIMAL(%d, %d)); }", precision, scale));
        ContiguousSet<Long> longValues = longsBetween(1, 1_000);
        ImmutableList.Builder<SqlDecimal> expectedValues = new ImmutableList.Builder<>();
        for (Long value : longValues) {
            expectedValues.add(SqlDecimal.of(value, precision, scale));
        }
        tester.testRoundTrip(javaLongObjectInspector, longValues, expectedValues.build(), createDecimalType(precision, scale), Optional.of(parquetSchema));
    }
}
Also used : ImmutableList(com.google.common.collect.ImmutableList) CacheBuilder(com.google.common.cache.CacheBuilder) SqlDecimal(com.facebook.presto.common.type.SqlDecimal) MessageType(org.apache.parquet.schema.MessageType) MessageTypeParser.parseMessageType(org.apache.parquet.schema.MessageTypeParser.parseMessageType) Test(org.testng.annotations.Test)

Example 4 with SqlDecimal

use of com.facebook.presto.common.type.SqlDecimal in project presto by prestodb.

the class AbstractTestOrcReader method decimalSequence.

private static List<SqlDecimal> decimalSequence(String start, String step, int items, int precision, int scale) {
    BigInteger decimalStep = new BigInteger(step);
    List<SqlDecimal> values = new ArrayList<>();
    BigInteger nextValue = new BigInteger(start);
    for (int i = 0; i < items; i++) {
        values.add(new SqlDecimal(nextValue, precision, scale));
        nextValue = nextValue.add(decimalStep);
    }
    return values;
}
Also used : Lists.newArrayList(com.google.common.collect.Lists.newArrayList) ArrayList(java.util.ArrayList) BigInteger(java.math.BigInteger) SqlDecimal(com.facebook.presto.common.type.SqlDecimal)

Example 5 with SqlDecimal

use of com.facebook.presto.common.type.SqlDecimal in project presto by prestodb.

the class AbstractTestRcFileReader method decimalSequence.

private static List<SqlDecimal> decimalSequence(String start, String step, int items, int precision, int scale) {
    BigInteger decimalStep = new BigInteger(step);
    List<SqlDecimal> values = new ArrayList<>();
    BigInteger nextValue = new BigInteger(start);
    for (int i = 0; i < items; i++) {
        values.add(new SqlDecimal(nextValue, precision, scale));
        nextValue = nextValue.add(decimalStep);
    }
    return values;
}
Also used : ArrayList(java.util.ArrayList) BigInteger(java.math.BigInteger) SqlDecimal(com.facebook.presto.common.type.SqlDecimal)

Aggregations

SqlDecimal (com.facebook.presto.common.type.SqlDecimal)21 ArrayList (java.util.ArrayList)10 DecimalType (com.facebook.presto.common.type.DecimalType)9 SqlVarbinary (com.facebook.presto.common.type.SqlVarbinary)9 ImmutableList (com.google.common.collect.ImmutableList)9 BigInteger (java.math.BigInteger)9 SqlTimestamp (com.facebook.presto.common.type.SqlTimestamp)7 Type (com.facebook.presto.common.type.Type)7 Slice (io.airlift.slice.Slice)7 BlockBuilder (com.facebook.presto.common.block.BlockBuilder)6 ArrayType (com.facebook.presto.common.type.ArrayType)6 SqlDate (com.facebook.presto.common.type.SqlDate)6 List (java.util.List)6 Map (java.util.Map)6 RowType (com.facebook.presto.common.type.RowType)5 VarcharType (com.facebook.presto.common.type.VarcharType)5 HashMap (java.util.HashMap)5 CharType (com.facebook.presto.common.type.CharType)4 MapType (com.facebook.presto.common.type.MapType)4 ImmutableMap (com.google.common.collect.ImmutableMap)4