Search in sources :

Example 1 with TINYINT

use of com.facebook.presto.spi.type.TinyintType.TINYINT in project presto by prestodb.

the class RcFileTester method preprocessWriteValueOld.

private static Object preprocessWriteValueOld(Type type, Object value) {
    if (value == null) {
        return null;
    }
    if (type.equals(BOOLEAN)) {
        return value;
    } else if (type.equals(TINYINT)) {
        return ((Number) value).byteValue();
    } else if (type.equals(SMALLINT)) {
        return ((Number) value).shortValue();
    } else if (type.equals(INTEGER)) {
        return ((Number) value).intValue();
    } else if (type.equals(BIGINT)) {
        return ((Number) value).longValue();
    } else if (type.equals(REAL)) {
        return ((Number) value).floatValue();
    } else if (type.equals(DOUBLE)) {
        return ((Number) value).doubleValue();
    } else if (type instanceof VarcharType) {
        return value;
    } else if (type.equals(VARBINARY)) {
        return ((SqlVarbinary) value).getBytes();
    } else if (type.equals(DATE)) {
        int days = ((SqlDate) value).getDays();
        LocalDate localDate = LocalDate.ofEpochDay(days);
        ZonedDateTime zonedDateTime = localDate.atStartOfDay(ZoneId.systemDefault());
        long millis = zonedDateTime.toEpochSecond() * 1000;
        Date date = new Date(0);
        // mills must be set separately to avoid masking
        date.setTime(millis);
        return date;
    } else if (type.equals(TIMESTAMP)) {
        long millisUtc = (int) ((SqlTimestamp) value).getMillisUtc();
        return new Timestamp(millisUtc);
    } else if (type instanceof DecimalType) {
        return HiveDecimal.create(((SqlDecimal) value).toBigDecimal());
    } else if (type.getTypeSignature().getBase().equals(ARRAY)) {
        Type elementType = type.getTypeParameters().get(0);
        return ((List<?>) value).stream().map(element -> preprocessWriteValueOld(elementType, element)).collect(toList());
    } else if (type.getTypeSignature().getBase().equals(MAP)) {
        Type keyType = type.getTypeParameters().get(0);
        Type valueType = type.getTypeParameters().get(1);
        Map<Object, Object> newMap = new HashMap<>();
        for (Entry<?, ?> entry : ((Map<?, ?>) value).entrySet()) {
            newMap.put(preprocessWriteValueOld(keyType, entry.getKey()), preprocessWriteValueOld(valueType, entry.getValue()));
        }
        return newMap;
    } else if (type.getTypeSignature().getBase().equals(ROW)) {
        List<?> fieldValues = (List<?>) value;
        List<Type> fieldTypes = type.getTypeParameters();
        List<Object> newStruct = new ArrayList<>();
        for (int fieldId = 0; fieldId < fieldValues.size(); fieldId++) {
            newStruct.add(preprocessWriteValueOld(fieldTypes.get(fieldId), fieldValues.get(fieldId)));
        }
        return newStruct;
    }
    throw new IllegalArgumentException("unsupported type: " + type);
}
Also used : SnappyCodec(org.apache.hadoop.io.compress.SnappyCodec) DateTimeZone(org.joda.time.DateTimeZone) LZ4(com.facebook.presto.rcfile.RcFileTester.Compression.LZ4) ZLIB(com.facebook.presto.rcfile.RcFileTester.Compression.ZLIB) ZonedDateTime(java.time.ZonedDateTime) PrimitiveObjectInspectorFactory.javaByteObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaByteObjectInspector) Text(org.apache.hadoop.io.Text) PrimitiveObjectInspectorFactory.javaLongObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaLongObjectInspector) SqlDate(com.facebook.presto.spi.type.SqlDate) DateWritable(org.apache.hadoop.hive.serde2.io.DateWritable) Writable(org.apache.hadoop.io.Writable) PrimitiveObjectInspectorFactory.javaTimestampObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaTimestampObjectInspector) RowType(com.facebook.presto.type.RowType) PrimitiveObjectInspectorFactory.javaDateObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaDateObjectInspector) BIGINT(com.facebook.presto.spi.type.BigintType.BIGINT) FileSplit(org.apache.hadoop.mapred.FileSplit) BOOLEAN(com.facebook.presto.spi.type.BooleanType.BOOLEAN) RCFileInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat) Slices(io.airlift.slice.Slices) Configuration(org.apache.hadoop.conf.Configuration) Map(java.util.Map) BigInteger(java.math.BigInteger) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) Assert.assertFalse(org.testng.Assert.assertFalse) LazyBinaryArray(org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryArray) IntWritable(org.apache.hadoop.io.IntWritable) SERIALIZATION_LIB(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB) PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector) Decimals(com.facebook.presto.spi.type.Decimals) BytesRefArrayWritable(org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable) TINYINT(com.facebook.presto.spi.type.TinyintType.TINYINT) META_TABLE_COLUMN_TYPES(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMN_TYPES) PrimitiveObjectInspectorFactory.javaFloatObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaFloatObjectInspector) LazyMap(org.apache.hadoop.hive.serde2.lazy.LazyMap) PrimitiveObjectInspectorFactory.javaDoubleObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaDoubleObjectInspector) LazyArray(org.apache.hadoop.hive.serde2.lazy.LazyArray) Set(java.util.Set) VARCHAR(com.facebook.presto.spi.type.VarcharType.VARCHAR) READ_ALL_COLUMNS(org.apache.hadoop.hive.serde2.ColumnProjectionUtils.READ_ALL_COLUMNS) ZoneId(java.time.ZoneId) LzoCodec(com.hadoop.compression.lzo.LzoCodec) ROW(com.facebook.presto.spi.type.StandardTypes.ROW) BooleanWritable(org.apache.hadoop.io.BooleanWritable) RecordReader(org.apache.hadoop.mapred.RecordReader) PrimitiveObjectInspectorFactory.javaIntObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaIntObjectInspector) SqlDecimal(com.facebook.presto.spi.type.SqlDecimal) StructField(org.apache.hadoop.hive.serde2.objectinspector.StructField) Lz4Codec(org.apache.hadoop.io.compress.Lz4Codec) Iterables(com.google.common.collect.Iterables) Decimals.rescale(com.facebook.presto.spi.type.Decimals.rescale) RcFileDecoderUtils.findFirstSyncPosition(com.facebook.presto.rcfile.RcFileDecoderUtils.findFirstSyncPosition) DOUBLE(com.facebook.presto.spi.type.DoubleType.DOUBLE) Slice(io.airlift.slice.Slice) MEGABYTE(io.airlift.units.DataSize.Unit.MEGABYTE) StructObject(org.apache.hadoop.hive.serde2.StructObject) UTC_KEY(com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY) Functions.constant(com.google.common.base.Functions.constant) MapType(com.facebook.presto.type.MapType) META_TABLE_COLUMNS(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMNS) PRESTO_RCFILE_WRITER_VERSION(com.facebook.presto.rcfile.RcFileWriter.PRESTO_RCFILE_WRITER_VERSION) ArrayList(java.util.ArrayList) ARRAY(com.facebook.presto.spi.type.StandardTypes.ARRAY) Lists(com.google.common.collect.Lists) PrimitiveObjectInspectorFactory.javaShortObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaShortObjectInspector) Type(com.facebook.presto.spi.type.Type) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) TIMESTAMP(com.facebook.presto.spi.type.TimestampType.TIMESTAMP) LinkedHashSet(java.util.LinkedHashSet) Properties(java.util.Properties) AbstractIterator(com.google.common.collect.AbstractIterator) Throwables(com.google.common.base.Throwables) FileOutputStream(java.io.FileOutputStream) IOException(java.io.IOException) ObjectInspectorFactory.getStandardStructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector) DecimalTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo) File(java.io.File) NULL(org.apache.hadoop.mapred.Reporter.NULL) SettableStructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector) BinaryRcFileEncoding(com.facebook.presto.rcfile.binary.BinaryRcFileEncoding) SqlVarbinary(com.facebook.presto.spi.type.SqlVarbinary) SIZE_OF_LONG(io.airlift.slice.SizeOf.SIZE_OF_LONG) Deserializer(org.apache.hadoop.hive.serde2.Deserializer) VarcharType(com.facebook.presto.spi.type.VarcharType) SerDeException(org.apache.hadoop.hive.serde2.SerDeException) FloatWritable(org.apache.hadoop.io.FloatWritable) Page(com.facebook.presto.spi.Page) RecordWriter(org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter) Block(com.facebook.presto.spi.block.Block) Iterables.transform(com.google.common.collect.Iterables.transform) LazyBinaryColumnarSerDe(org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe) GzipCodec(org.apache.hadoop.io.compress.GzipCodec) LongWritable(org.apache.hadoop.io.LongWritable) SqlTimestamp(com.facebook.presto.spi.type.SqlTimestamp) DecimalType(com.facebook.presto.spi.type.DecimalType) SESSION(com.facebook.presto.testing.TestingConnectorSession.SESSION) InputFormat(org.apache.hadoop.mapred.InputFormat) Path(org.apache.hadoop.fs.Path) KILOBYTE(io.airlift.units.DataSize.Unit.KILOBYTE) NONE(com.facebook.presto.rcfile.RcFileTester.Compression.NONE) ShortWritable(org.apache.hadoop.hive.serde2.io.ShortWritable) FileUtils.deleteRecursively(io.airlift.testing.FileUtils.deleteRecursively) SIZE_OF_INT(io.airlift.slice.SizeOf.SIZE_OF_INT) ImmutableSet(com.google.common.collect.ImmutableSet) Files.createTempDir(com.google.common.io.Files.createTempDir) ImmutableMap(com.google.common.collect.ImmutableMap) Timestamp(java.sql.Timestamp) RCFileOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat) BlockBuilder(com.facebook.presto.spi.block.BlockBuilder) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) ColumnarSerDe(org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe) Objects(java.util.Objects) DataSize(io.airlift.units.DataSize) List(java.util.List) LocalDate(java.time.LocalDate) Entry(java.util.Map.Entry) Optional(java.util.Optional) READ_COLUMN_IDS_CONF_STR(org.apache.hadoop.hive.serde2.ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR) INTEGER(com.facebook.presto.spi.type.IntegerType.INTEGER) LazyPrimitive(org.apache.hadoop.hive.serde2.lazy.LazyPrimitive) Assert.assertNull(org.testng.Assert.assertNull) ArrayType(com.facebook.presto.type.ArrayType) LazyBinaryMap(org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryMap) PrimitiveObjectInspectorFactory.javaBooleanObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaBooleanObjectInspector) Assert.assertEquals(org.testng.Assert.assertEquals) HashMap(java.util.HashMap) DoubleWritable(org.apache.hadoop.io.DoubleWritable) PRESTO_RCFILE_WRITER_VERSION_METADATA_KEY(com.facebook.presto.rcfile.RcFileWriter.PRESTO_RCFILE_WRITER_VERSION_METADATA_KEY) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector) MAP(com.facebook.presto.spi.type.StandardTypes.MAP) OutputStreamSliceOutput(io.airlift.slice.OutputStreamSliceOutput) COMPRESS_CODEC(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.COMPRESS_CODEC) ImmutableList(com.google.common.collect.ImmutableList) ByteWritable(org.apache.hadoop.io.ByteWritable) BlockBuilderStatus(com.facebook.presto.spi.block.BlockBuilderStatus) BytesWritable(org.apache.hadoop.io.BytesWritable) TimestampWritable(org.apache.hadoop.hive.serde2.io.TimestampWritable) Math.toIntExact(java.lang.Math.toIntExact) SNAPPY(com.facebook.presto.rcfile.RcFileTester.Compression.SNAPPY) VARBINARY(com.facebook.presto.spi.type.VarbinaryType.VARBINARY) Iterator(java.util.Iterator) Iterators.advance(com.google.common.collect.Iterators.advance) HadoopNative(com.facebook.presto.hadoop.HadoopNative) SMALLINT(com.facebook.presto.spi.type.SmallintType.SMALLINT) FileInputStream(java.io.FileInputStream) Date(java.sql.Date) TextRcFileEncoding(com.facebook.presto.rcfile.text.TextRcFileEncoding) JobConf(org.apache.hadoop.mapred.JobConf) Collectors.toList(java.util.stream.Collectors.toList) ObjectInspectorFactory(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory) DATE(com.facebook.presto.spi.type.DateType.DATE) REAL(com.facebook.presto.spi.type.RealType.REAL) Serializer(org.apache.hadoop.hive.serde2.Serializer) HiveDecimal(org.apache.hadoop.hive.common.type.HiveDecimal) Closeable(java.io.Closeable) Assert.assertTrue(org.testng.Assert.assertTrue) PrimitiveObjectInspectorFactory.javaStringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaStringObjectInspector) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) Collections(java.util.Collections) BYTE(io.airlift.units.DataSize.Unit.BYTE) InputStream(java.io.InputStream) VarcharType(com.facebook.presto.spi.type.VarcharType) SqlVarbinary(com.facebook.presto.spi.type.SqlVarbinary) ArrayList(java.util.ArrayList) SqlDecimal(com.facebook.presto.spi.type.SqlDecimal) LocalDate(java.time.LocalDate) SqlTimestamp(com.facebook.presto.spi.type.SqlTimestamp) Timestamp(java.sql.Timestamp) SqlDate(com.facebook.presto.spi.type.SqlDate) LocalDate(java.time.LocalDate) Date(java.sql.Date) RowType(com.facebook.presto.type.RowType) MapType(com.facebook.presto.type.MapType) Type(com.facebook.presto.spi.type.Type) VarcharType(com.facebook.presto.spi.type.VarcharType) DecimalType(com.facebook.presto.spi.type.DecimalType) ArrayType(com.facebook.presto.type.ArrayType) Entry(java.util.Map.Entry) ZonedDateTime(java.time.ZonedDateTime) DecimalType(com.facebook.presto.spi.type.DecimalType) ArrayList(java.util.ArrayList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) Collectors.toList(java.util.stream.Collectors.toList) StructObject(org.apache.hadoop.hive.serde2.StructObject) Map(java.util.Map) LazyMap(org.apache.hadoop.hive.serde2.lazy.LazyMap) ImmutableMap(com.google.common.collect.ImmutableMap) LazyBinaryMap(org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryMap) HashMap(java.util.HashMap)

Aggregations

HadoopNative (com.facebook.presto.hadoop.HadoopNative)1 RcFileDecoderUtils.findFirstSyncPosition (com.facebook.presto.rcfile.RcFileDecoderUtils.findFirstSyncPosition)1 LZ4 (com.facebook.presto.rcfile.RcFileTester.Compression.LZ4)1 NONE (com.facebook.presto.rcfile.RcFileTester.Compression.NONE)1 SNAPPY (com.facebook.presto.rcfile.RcFileTester.Compression.SNAPPY)1 ZLIB (com.facebook.presto.rcfile.RcFileTester.Compression.ZLIB)1 PRESTO_RCFILE_WRITER_VERSION (com.facebook.presto.rcfile.RcFileWriter.PRESTO_RCFILE_WRITER_VERSION)1 PRESTO_RCFILE_WRITER_VERSION_METADATA_KEY (com.facebook.presto.rcfile.RcFileWriter.PRESTO_RCFILE_WRITER_VERSION_METADATA_KEY)1 BinaryRcFileEncoding (com.facebook.presto.rcfile.binary.BinaryRcFileEncoding)1 TextRcFileEncoding (com.facebook.presto.rcfile.text.TextRcFileEncoding)1 Page (com.facebook.presto.spi.Page)1 Block (com.facebook.presto.spi.block.Block)1 BlockBuilder (com.facebook.presto.spi.block.BlockBuilder)1 BlockBuilderStatus (com.facebook.presto.spi.block.BlockBuilderStatus)1 BIGINT (com.facebook.presto.spi.type.BigintType.BIGINT)1 BOOLEAN (com.facebook.presto.spi.type.BooleanType.BOOLEAN)1 DATE (com.facebook.presto.spi.type.DateType.DATE)1 DecimalType (com.facebook.presto.spi.type.DecimalType)1 Decimals (com.facebook.presto.spi.type.Decimals)1 Decimals.rescale (com.facebook.presto.spi.type.Decimals.rescale)1