Search in sources :

Example 1 with ROW

use of io.prestosql.spi.type.StandardTypes.ROW in project hetu-core by openlookeng.

the class RcFileTester method preprocessWriteValueOld.

private static Object preprocessWriteValueOld(Format format, Type type, Object value) {
    if (value == null) {
        return null;
    }
    if (type.equals(BOOLEAN)) {
        return value;
    }
    if (type.equals(TINYINT)) {
        return ((Number) value).byteValue();
    }
    if (type.equals(SMALLINT)) {
        return ((Number) value).shortValue();
    }
    if (type.equals(INTEGER)) {
        return ((Number) value).intValue();
    }
    if (type.equals(BIGINT)) {
        return ((Number) value).longValue();
    }
    if (type.equals(REAL)) {
        return ((Number) value).floatValue();
    }
    if (type.equals(DOUBLE)) {
        return ((Number) value).doubleValue();
    }
    if (type instanceof VarcharType) {
        return value;
    }
    if (type.equals(VARBINARY)) {
        return ((SqlVarbinary) value).getBytes();
    }
    if (type.equals(DATE)) {
        return Date.ofEpochDay(((SqlDate) value).getDays());
    }
    if (type.equals(TIMESTAMP)) {
        long millis = ((SqlTimestamp) value).getMillis();
        if (format == Format.BINARY) {
            millis = HIVE_STORAGE_TIME_ZONE.convertLocalToUTC(millis, false);
        }
        return Timestamp.ofEpochMilli(millis);
    }
    if (type instanceof DecimalType) {
        return HiveDecimal.create(((SqlDecimal) value).toBigDecimal());
    }
    if (type.getTypeSignature().getBase().equals(ARRAY)) {
        Type elementType = type.getTypeParameters().get(0);
        return ((List<?>) value).stream().map(element -> preprocessWriteValueOld(format, elementType, element)).collect(toList());
    }
    if (type.getTypeSignature().getBase().equals(MAP)) {
        Type keyType = type.getTypeParameters().get(0);
        Type valueType = type.getTypeParameters().get(1);
        Map<Object, Object> newMap = new HashMap<>();
        for (Entry<?, ?> entry : ((Map<?, ?>) value).entrySet()) {
            newMap.put(preprocessWriteValueOld(format, keyType, entry.getKey()), preprocessWriteValueOld(format, valueType, entry.getValue()));
        }
        return newMap;
    }
    if (type.getTypeSignature().getBase().equals(ROW)) {
        List<?> fieldValues = (List<?>) value;
        List<Type> fieldTypes = type.getTypeParameters();
        List<Object> newStruct = new ArrayList<>();
        for (int fieldId = 0; fieldId < fieldValues.size(); fieldId++) {
            newStruct.add(preprocessWriteValueOld(format, fieldTypes.get(fieldId), fieldValues.get(fieldId)));
        }
        return newStruct;
    }
    throw new IllegalArgumentException("unsupported type: " + type);
}
Also used : SnappyCodec(org.apache.hadoop.io.compress.SnappyCodec) DateTimeZone(org.joda.time.DateTimeZone) PrimitiveObjectInspectorFactory.javaByteObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaByteObjectInspector) Text(org.apache.hadoop.io.Text) PrimitiveObjectInspectorFactory.javaLongObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaLongObjectInspector) Writable(org.apache.hadoop.io.Writable) PrimitiveObjectInspectorFactory.javaTimestampObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaTimestampObjectInspector) Date(org.apache.hadoop.hive.common.type.Date) PrimitiveObjectInspectorFactory.javaDateObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaDateObjectInspector) FileSplit(org.apache.hadoop.mapred.FileSplit) RCFileInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat) Files.createTempDirectory(java.nio.file.Files.createTempDirectory) Slices(io.airlift.slice.Slices) Configuration(org.apache.hadoop.conf.Configuration) Map(java.util.Map) RowType(io.prestosql.spi.type.RowType) NONE(io.prestosql.rcfile.RcFileTester.Compression.NONE) BigInteger(java.math.BigInteger) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) Assert.assertFalse(org.testng.Assert.assertFalse) LazyBinaryArray(org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryArray) IntWritable(org.apache.hadoop.io.IntWritable) SERIALIZATION_LIB(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB) PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector) BytesRefArrayWritable(org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable) META_TABLE_COLUMN_TYPES(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMN_TYPES) PrimitiveObjectInspectorFactory.javaFloatObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaFloatObjectInspector) LazyMap(org.apache.hadoop.hive.serde2.lazy.LazyMap) PrimitiveObjectInspectorFactory.javaDoubleObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaDoubleObjectInspector) LazyArray(org.apache.hadoop.hive.serde2.lazy.LazyArray) Set(java.util.Set) READ_ALL_COLUMNS(org.apache.hadoop.hive.serde2.ColumnProjectionUtils.READ_ALL_COLUMNS) TIMESTAMP(io.prestosql.spi.type.TimestampType.TIMESTAMP) Metadata(io.prestosql.metadata.Metadata) UncheckedIOException(java.io.UncheckedIOException) LzoCodec(com.hadoop.compression.lzo.LzoCodec) BooleanWritable(org.apache.hadoop.io.BooleanWritable) RecordReader(org.apache.hadoop.mapred.RecordReader) PrimitiveObjectInspectorFactory.javaIntObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaIntObjectInspector) StructField(org.apache.hadoop.hive.serde2.objectinspector.StructField) Lz4Codec(org.apache.hadoop.io.compress.Lz4Codec) Iterables(com.google.common.collect.Iterables) Slice(io.airlift.slice.Slice) MEGABYTE(io.airlift.units.DataSize.Unit.MEGABYTE) StructObject(org.apache.hadoop.hive.serde2.StructObject) Functions.constant(com.google.common.base.Functions.constant) TypeSignatureParameter(io.prestosql.spi.type.TypeSignatureParameter) META_TABLE_COLUMNS(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMNS) Decimals.rescale(io.prestosql.spi.type.Decimals.rescale) ArrayList(java.util.ArrayList) MapType(io.prestosql.spi.type.MapType) Lists(com.google.common.collect.Lists) ALLOW_INSECURE(com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE) VARCHAR(io.prestosql.spi.type.VarcharType.VARCHAR) PrimitiveObjectInspectorFactory.javaShortObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaShortObjectInspector) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) DOUBLE(io.prestosql.spi.type.DoubleType.DOUBLE) ROW(io.prestosql.spi.type.StandardTypes.ROW) LinkedHashSet(java.util.LinkedHashSet) HadoopNative(io.prestosql.hadoop.HadoopNative) Properties(java.util.Properties) RcFileDecoderUtils.findFirstSyncPosition(io.prestosql.rcfile.RcFileDecoderUtils.findFirstSyncPosition) AbstractIterator(com.google.common.collect.AbstractIterator) FileOutputStream(java.io.FileOutputStream) IOException(java.io.IOException) ObjectInspectorFactory.getStandardStructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector) DecimalTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo) File(java.io.File) NULL(org.apache.hadoop.mapred.Reporter.NULL) SettableStructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector) VARBINARY(io.prestosql.spi.type.VarbinaryType.VARBINARY) DateTimeTestingUtils.sqlTimestampOf(io.prestosql.testing.DateTimeTestingUtils.sqlTimestampOf) SIZE_OF_LONG(io.airlift.slice.SizeOf.SIZE_OF_LONG) Deserializer(org.apache.hadoop.hive.serde2.Deserializer) SerDeException(org.apache.hadoop.hive.serde2.SerDeException) FloatWritable(org.apache.hadoop.io.FloatWritable) VarcharType(io.prestosql.spi.type.VarcharType) RecordWriter(org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter) Iterables.transform(com.google.common.collect.Iterables.transform) LazyBinaryColumnarSerDe(org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe) MoreFiles.deleteRecursively(com.google.common.io.MoreFiles.deleteRecursively) DecimalType(io.prestosql.spi.type.DecimalType) GzipCodec(org.apache.hadoop.io.compress.GzipCodec) LongWritable(org.apache.hadoop.io.LongWritable) TimestampWritableV2(org.apache.hadoop.hive.serde2.io.TimestampWritableV2) InputFormat(org.apache.hadoop.mapred.InputFormat) Path(org.apache.hadoop.fs.Path) BOOLEAN(io.prestosql.spi.type.BooleanType.BOOLEAN) KILOBYTE(io.airlift.units.DataSize.Unit.KILOBYTE) Type(io.prestosql.spi.type.Type) ShortWritable(org.apache.hadoop.hive.serde2.io.ShortWritable) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) SIZE_OF_INT(io.airlift.slice.SizeOf.SIZE_OF_INT) ARRAY(io.prestosql.spi.type.StandardTypes.ARRAY) ImmutableSet(com.google.common.collect.ImmutableSet) DateWritableV2(org.apache.hadoop.hive.serde2.io.DateWritableV2) ImmutableMap(com.google.common.collect.ImmutableMap) BlockBuilder(io.prestosql.spi.block.BlockBuilder) MetadataManager.createTestMetadataManager(io.prestosql.metadata.MetadataManager.createTestMetadataManager) Collections.nCopies(java.util.Collections.nCopies) ArrayType(io.prestosql.spi.type.ArrayType) RCFileOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat) TINYINT(io.prestosql.spi.type.TinyintType.TINYINT) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) ColumnarSerDe(org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe) Objects(java.util.Objects) DataSize(io.airlift.units.DataSize) List(java.util.List) PRESTO_RCFILE_WRITER_VERSION_METADATA_KEY(io.prestosql.rcfile.RcFileWriter.PRESTO_RCFILE_WRITER_VERSION_METADATA_KEY) ZLIB(io.prestosql.rcfile.RcFileTester.Compression.ZLIB) Entry(java.util.Map.Entry) Optional(java.util.Optional) READ_COLUMN_IDS_CONF_STR(org.apache.hadoop.hive.serde2.ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR) SqlTimestamp(io.prestosql.spi.type.SqlTimestamp) LazyPrimitive(org.apache.hadoop.hive.serde2.lazy.LazyPrimitive) Assert.assertNull(org.testng.Assert.assertNull) SESSION(io.prestosql.testing.TestingConnectorSession.SESSION) TextRcFileEncoding(io.prestosql.rcfile.text.TextRcFileEncoding) LazyBinaryMap(org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryMap) PrimitiveObjectInspectorFactory.javaBooleanObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaBooleanObjectInspector) SqlDate(io.prestosql.spi.type.SqlDate) Assert.assertEquals(org.testng.Assert.assertEquals) Decimals(io.prestosql.spi.type.Decimals) HashMap(java.util.HashMap) INTEGER(io.prestosql.spi.type.IntegerType.INTEGER) SqlDecimal(io.prestosql.spi.type.SqlDecimal) DoubleWritable(org.apache.hadoop.io.DoubleWritable) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector) SqlVarbinary(io.prestosql.spi.type.SqlVarbinary) OutputStreamSliceOutput(io.airlift.slice.OutputStreamSliceOutput) COMPRESS_CODEC(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.COMPRESS_CODEC) PRESTO_RCFILE_WRITER_VERSION(io.prestosql.rcfile.RcFileWriter.PRESTO_RCFILE_WRITER_VERSION) ImmutableList(com.google.common.collect.ImmutableList) ByteWritable(org.apache.hadoop.io.ByteWritable) MAP(io.prestosql.spi.type.StandardTypes.MAP) BytesWritable(org.apache.hadoop.io.BytesWritable) DATE(io.prestosql.spi.type.DateType.DATE) REAL(io.prestosql.spi.type.RealType.REAL) Math.toIntExact(java.lang.Math.toIntExact) Block(io.prestosql.spi.block.Block) Iterator(java.util.Iterator) Timestamp(org.apache.hadoop.hive.common.type.Timestamp) Iterators.advance(com.google.common.collect.Iterators.advance) Page(io.prestosql.spi.Page) FileInputStream(java.io.FileInputStream) BinaryRcFileEncoding(io.prestosql.rcfile.binary.BinaryRcFileEncoding) BZIP2(io.prestosql.rcfile.RcFileTester.Compression.BZIP2) LZ4(io.prestosql.rcfile.RcFileTester.Compression.LZ4) JobConf(org.apache.hadoop.mapred.JobConf) BZip2Codec(org.apache.hadoop.io.compress.BZip2Codec) Collectors.toList(java.util.stream.Collectors.toList) ObjectInspectorFactory(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory) SMALLINT(io.prestosql.spi.type.SmallintType.SMALLINT) Serializer(org.apache.hadoop.hive.serde2.Serializer) HiveDecimal(org.apache.hadoop.hive.common.type.HiveDecimal) Closeable(java.io.Closeable) Assert.assertTrue(org.testng.Assert.assertTrue) SNAPPY(io.prestosql.rcfile.RcFileTester.Compression.SNAPPY) PrimitiveObjectInspectorFactory.javaStringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaStringObjectInspector) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) Collections(java.util.Collections) BYTE(io.airlift.units.DataSize.Unit.BYTE) InputStream(java.io.InputStream) VarcharType(io.prestosql.spi.type.VarcharType) HashMap(java.util.HashMap) SqlVarbinary(io.prestosql.spi.type.SqlVarbinary) ArrayList(java.util.ArrayList) SqlTimestamp(io.prestosql.spi.type.SqlTimestamp) RowType(io.prestosql.spi.type.RowType) MapType(io.prestosql.spi.type.MapType) VarcharType(io.prestosql.spi.type.VarcharType) DecimalType(io.prestosql.spi.type.DecimalType) Type(io.prestosql.spi.type.Type) ArrayType(io.prestosql.spi.type.ArrayType) DecimalType(io.prestosql.spi.type.DecimalType) ArrayList(java.util.ArrayList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) Collectors.toList(java.util.stream.Collectors.toList) StructObject(org.apache.hadoop.hive.serde2.StructObject) Map(java.util.Map) LazyMap(org.apache.hadoop.hive.serde2.lazy.LazyMap) ImmutableMap(com.google.common.collect.ImmutableMap) LazyBinaryMap(org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryMap) HashMap(java.util.HashMap)

Example 2 with ROW

use of io.prestosql.spi.type.StandardTypes.ROW in project hetu-core by openlookeng.

the class TestRowParametricType method testTypeSignatureRoundTrip.

@Test
public void testTypeSignatureRoundTrip() {
    TypeManager typeManager = new InternalTypeManager(createTestMetadataManager().getFunctionAndTypeManager());
    TypeSignature typeSignature = new TypeSignature(ROW, TypeSignatureParameter.of(new NamedTypeSignature(Optional.of(new RowFieldName("col1", false)), new TypeSignature(BIGINT))), TypeSignatureParameter.of(new NamedTypeSignature(Optional.of(new RowFieldName("col2", true)), new TypeSignature(DOUBLE))));
    List<TypeParameter> parameters = typeSignature.getParameters().stream().map(parameter -> TypeParameter.of(parameter, typeManager)).collect(Collectors.toList());
    Type rowType = RowParametricType.ROW.createType(typeManager, parameters);
    assertEquals(rowType.getTypeSignature(), typeSignature);
}
Also used : DOUBLE(io.prestosql.spi.type.StandardTypes.DOUBLE) MetadataManager.createTestMetadataManager(io.prestosql.metadata.MetadataManager.createTestMetadataManager) TypeManager(io.prestosql.spi.type.TypeManager) Assert.assertEquals(org.testng.Assert.assertEquals) TypeSignatureParameter(io.prestosql.spi.type.TypeSignatureParameter) Test(org.testng.annotations.Test) Collectors(java.util.stream.Collectors) RowParametricType(io.prestosql.spi.type.RowParametricType) NamedTypeSignature(io.prestosql.spi.type.NamedTypeSignature) List(java.util.List) TypeParameter(io.prestosql.spi.type.TypeParameter) Optional(java.util.Optional) RowFieldName(io.prestosql.spi.type.RowFieldName) Type(io.prestosql.spi.type.Type) TypeSignature(io.prestosql.spi.type.TypeSignature) ROW(io.prestosql.spi.type.StandardTypes.ROW) BIGINT(io.prestosql.spi.type.StandardTypes.BIGINT) NamedTypeSignature(io.prestosql.spi.type.NamedTypeSignature) TypeSignature(io.prestosql.spi.type.TypeSignature) TypeParameter(io.prestosql.spi.type.TypeParameter) RowParametricType(io.prestosql.spi.type.RowParametricType) Type(io.prestosql.spi.type.Type) RowFieldName(io.prestosql.spi.type.RowFieldName) TypeManager(io.prestosql.spi.type.TypeManager) NamedTypeSignature(io.prestosql.spi.type.NamedTypeSignature) Test(org.testng.annotations.Test)

Aggregations

MetadataManager.createTestMetadataManager (io.prestosql.metadata.MetadataManager.createTestMetadataManager)2 Functions.constant (com.google.common.base.Functions.constant)1 AbstractIterator (com.google.common.collect.AbstractIterator)1 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 ImmutableSet (com.google.common.collect.ImmutableSet)1 Iterables (com.google.common.collect.Iterables)1 Iterables.transform (com.google.common.collect.Iterables.transform)1 Iterators.advance (com.google.common.collect.Iterators.advance)1 Lists (com.google.common.collect.Lists)1 MoreFiles.deleteRecursively (com.google.common.io.MoreFiles.deleteRecursively)1 ALLOW_INSECURE (com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE)1 LzoCodec (com.hadoop.compression.lzo.LzoCodec)1 OutputStreamSliceOutput (io.airlift.slice.OutputStreamSliceOutput)1 SIZE_OF_INT (io.airlift.slice.SizeOf.SIZE_OF_INT)1 SIZE_OF_LONG (io.airlift.slice.SizeOf.SIZE_OF_LONG)1 Slice (io.airlift.slice.Slice)1 Slices (io.airlift.slice.Slices)1 DataSize (io.airlift.units.DataSize)1 BYTE (io.airlift.units.DataSize.Unit.BYTE)1