Search in sources :

Example 66 with DecimalType

use of io.trino.spi.type.DecimalType in project trino by trinodb.

the class HivePageSource method createCoercer.

private static Optional<Function<Block, Block>> createCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) {
    if (fromHiveType.equals(toHiveType)) {
        return Optional.empty();
    }
    Type fromType = fromHiveType.getType(typeManager);
    Type toType = toHiveType.getType(typeManager);
    if (toType instanceof VarcharType && (fromHiveType.equals(HIVE_BYTE) || fromHiveType.equals(HIVE_SHORT) || fromHiveType.equals(HIVE_INT) || fromHiveType.equals(HIVE_LONG))) {
        return Optional.of(new IntegerNumberToVarcharCoercer<>(fromType, (VarcharType) toType));
    }
    if (fromType instanceof VarcharType && (toHiveType.equals(HIVE_BYTE) || toHiveType.equals(HIVE_SHORT) || toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG))) {
        return Optional.of(new VarcharToIntegerNumberCoercer<>((VarcharType) fromType, toType));
    }
    if (fromType instanceof VarcharType && toType instanceof VarcharType) {
        VarcharType toVarcharType = (VarcharType) toType;
        VarcharType fromVarcharType = (VarcharType) fromType;
        if (narrowerThan(toVarcharType, fromVarcharType)) {
            return Optional.of(new VarcharCoercer(fromVarcharType, toVarcharType));
        }
        return Optional.empty();
    }
    if (fromHiveType.equals(HIVE_BYTE) && (toHiveType.equals(HIVE_SHORT) || toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG))) {
        return Optional.of(new IntegerNumberUpscaleCoercer<>(fromType, toType));
    }
    if (fromHiveType.equals(HIVE_SHORT) && (toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG))) {
        return Optional.of(new IntegerNumberUpscaleCoercer<>(fromType, toType));
    }
    if (fromHiveType.equals(HIVE_INT) && toHiveType.equals(HIVE_LONG)) {
        return Optional.of(new IntegerNumberUpscaleCoercer<>(fromType, toType));
    }
    if (fromHiveType.equals(HIVE_FLOAT) && toHiveType.equals(HIVE_DOUBLE)) {
        return Optional.of(new FloatToDoubleCoercer());
    }
    if (fromHiveType.equals(HIVE_DOUBLE) && toHiveType.equals(HIVE_FLOAT)) {
        return Optional.of(new DoubleToFloatCoercer());
    }
    if (fromType instanceof DecimalType && toType instanceof DecimalType) {
        return Optional.of(createDecimalToDecimalCoercer((DecimalType) fromType, (DecimalType) toType));
    }
    if (fromType instanceof DecimalType && toType == DOUBLE) {
        return Optional.of(createDecimalToDoubleCoercer((DecimalType) fromType));
    }
    if (fromType instanceof DecimalType && toType == REAL) {
        return Optional.of(createDecimalToRealCoercer((DecimalType) fromType));
    }
    if (fromType == DOUBLE && toType instanceof DecimalType) {
        return Optional.of(createDoubleToDecimalCoercer((DecimalType) toType));
    }
    if (fromType == REAL && toType instanceof DecimalType) {
        return Optional.of(createRealToDecimalCoercer((DecimalType) toType));
    }
    if (isArrayType(fromType) && isArrayType(toType)) {
        return Optional.of(new ListCoercer(typeManager, fromHiveType, toHiveType));
    }
    if (isMapType(fromType) && isMapType(toType)) {
        return Optional.of(new MapCoercer(typeManager, fromHiveType, toHiveType));
    }
    if (isRowType(fromType) && isRowType(toType)) {
        return Optional.of(new StructCoercer(typeManager, fromHiveType, toHiveType));
    }
    throw new TrinoException(NOT_SUPPORTED, format("Unsupported coercion from %s to %s", fromHiveType, toHiveType));
}
Also used : VarcharType(io.trino.spi.type.VarcharType) DoubleToFloatCoercer(io.trino.plugin.hive.coercions.DoubleToFloatCoercer) FloatToDoubleCoercer(io.trino.plugin.hive.coercions.FloatToDoubleCoercer) HiveUtil.isMapType(io.trino.plugin.hive.util.HiveUtil.isMapType) DecimalType(io.trino.spi.type.DecimalType) HiveUtil.isRowType(io.trino.plugin.hive.util.HiveUtil.isRowType) Type(io.trino.spi.type.Type) VarcharType(io.trino.spi.type.VarcharType) HiveUtil.isArrayType(io.trino.plugin.hive.util.HiveUtil.isArrayType) MapType(io.trino.spi.type.MapType) IntegerNumberToVarcharCoercer(io.trino.plugin.hive.coercions.IntegerNumberToVarcharCoercer) VarcharCoercer(io.trino.plugin.hive.coercions.VarcharCoercer) DecimalType(io.trino.spi.type.DecimalType) TrinoException(io.trino.spi.TrinoException)

Example 67 with DecimalType

use of io.trino.spi.type.DecimalType in project trino by trinodb.

the class MySqlClient method toWriteMapping.

@Override
public WriteMapping toWriteMapping(ConnectorSession session, Type type) {
    if (type == BOOLEAN) {
        return WriteMapping.booleanMapping("boolean", booleanWriteFunction());
    }
    if (type == TINYINT) {
        return WriteMapping.longMapping("tinyint", tinyintWriteFunction());
    }
    if (type == SMALLINT) {
        return WriteMapping.longMapping("smallint", smallintWriteFunction());
    }
    if (type == INTEGER) {
        return WriteMapping.longMapping("integer", integerWriteFunction());
    }
    if (type == BIGINT) {
        return WriteMapping.longMapping("bigint", bigintWriteFunction());
    }
    if (type == REAL) {
        return WriteMapping.longMapping("float", realWriteFunction());
    }
    if (type == DOUBLE) {
        return WriteMapping.doubleMapping("double precision", doubleWriteFunction());
    }
    if (type instanceof DecimalType) {
        DecimalType decimalType = (DecimalType) type;
        String dataType = format("decimal(%s, %s)", decimalType.getPrecision(), decimalType.getScale());
        if (decimalType.isShort()) {
            return WriteMapping.longMapping(dataType, shortDecimalWriteFunction(decimalType));
        }
        return WriteMapping.objectMapping(dataType, longDecimalWriteFunction(decimalType));
    }
    if (type == DATE) {
        return WriteMapping.longMapping("date", dateWriteFunctionUsingLocalDate());
    }
    if (type instanceof TimeType) {
        TimeType timeType = (TimeType) type;
        if (timeType.getPrecision() <= MAX_SUPPORTED_DATE_TIME_PRECISION) {
            return WriteMapping.longMapping(format("time(%s)", timeType.getPrecision()), timeWriteFunction(timeType.getPrecision()));
        }
        return WriteMapping.longMapping(format("time(%s)", MAX_SUPPORTED_DATE_TIME_PRECISION), timeWriteFunction(MAX_SUPPORTED_DATE_TIME_PRECISION));
    }
    if (TIME_WITH_TIME_ZONE.equals(type) || TIMESTAMP_TZ_MILLIS.equals(type)) {
        throw new TrinoException(NOT_SUPPORTED, "Unsupported column type: " + type.getDisplayName());
    }
    if (type instanceof TimestampType) {
        TimestampType timestampType = (TimestampType) type;
        if (timestampType.getPrecision() <= MAX_SUPPORTED_DATE_TIME_PRECISION) {
            verify(timestampType.getPrecision() <= TimestampType.MAX_SHORT_PRECISION);
            return WriteMapping.longMapping(format("datetime(%s)", timestampType.getPrecision()), timestampWriteFunction(timestampType));
        }
        return WriteMapping.objectMapping(format("datetime(%s)", MAX_SUPPORTED_DATE_TIME_PRECISION), longTimestampWriteFunction(timestampType, MAX_SUPPORTED_DATE_TIME_PRECISION));
    }
    if (VARBINARY.equals(type)) {
        return WriteMapping.sliceMapping("mediumblob", varbinaryWriteFunction());
    }
    if (type instanceof CharType) {
        return WriteMapping.sliceMapping("char(" + ((CharType) type).getLength() + ")", charWriteFunction());
    }
    if (type instanceof VarcharType) {
        VarcharType varcharType = (VarcharType) type;
        String dataType;
        if (varcharType.isUnbounded()) {
            dataType = "longtext";
        } else if (varcharType.getBoundedLength() <= 255) {
            dataType = "tinytext";
        } else if (varcharType.getBoundedLength() <= 65535) {
            dataType = "text";
        } else if (varcharType.getBoundedLength() <= 16777215) {
            dataType = "mediumtext";
        } else {
            dataType = "longtext";
        }
        return WriteMapping.sliceMapping(dataType, varcharWriteFunction());
    }
    if (type.equals(jsonType)) {
        return WriteMapping.sliceMapping("json", varcharWriteFunction());
    }
    throw new TrinoException(NOT_SUPPORTED, "Unsupported column type: " + type.getDisplayName());
}
Also used : VarcharType(io.trino.spi.type.VarcharType) DecimalType.createDecimalType(io.trino.spi.type.DecimalType.createDecimalType) DecimalType(io.trino.spi.type.DecimalType) TrinoException(io.trino.spi.TrinoException) TimestampType(io.trino.spi.type.TimestampType) TimestampType.createTimestampType(io.trino.spi.type.TimestampType.createTimestampType) CharType(io.trino.spi.type.CharType) TimeType(io.trino.spi.type.TimeType) TimeType.createTimeType(io.trino.spi.type.TimeType.createTimeType)

Example 68 with DecimalType

use of io.trino.spi.type.DecimalType in project trino by trinodb.

the class MongoPageSink method getObjectValue.

private Object getObjectValue(Type type, Block block, int position) {
    if (block.isNull(position)) {
        if (type.equals(OBJECT_ID)) {
            return new ObjectId();
        }
        return null;
    }
    if (type.equals(OBJECT_ID)) {
        return new ObjectId(block.getSlice(position, 0, block.getSliceLength(position)).getBytes());
    }
    if (type.equals(BooleanType.BOOLEAN)) {
        return type.getBoolean(block, position);
    }
    if (type.equals(BigintType.BIGINT)) {
        return type.getLong(block, position);
    }
    if (type.equals(IntegerType.INTEGER)) {
        return toIntExact(type.getLong(block, position));
    }
    if (type.equals(SmallintType.SMALLINT)) {
        return Shorts.checkedCast(type.getLong(block, position));
    }
    if (type.equals(TinyintType.TINYINT)) {
        return SignedBytes.checkedCast(type.getLong(block, position));
    }
    if (type.equals(RealType.REAL)) {
        return intBitsToFloat(toIntExact(type.getLong(block, position)));
    }
    if (type.equals(DoubleType.DOUBLE)) {
        return type.getDouble(block, position);
    }
    if (type instanceof VarcharType) {
        return type.getSlice(block, position).toStringUtf8();
    }
    if (type instanceof CharType) {
        return padSpaces(type.getSlice(block, position), ((CharType) type)).toStringUtf8();
    }
    if (type.equals(VarbinaryType.VARBINARY)) {
        return new Binary(type.getSlice(block, position).getBytes());
    }
    if (type.equals(DateType.DATE)) {
        long days = type.getLong(block, position);
        return new Date(TimeUnit.DAYS.toMillis(days));
    }
    if (type.equals(TimeType.TIME)) {
        long picos = type.getLong(block, position);
        return new Date(roundDiv(picos, PICOSECONDS_PER_MILLISECOND));
    }
    if (type.equals(TIMESTAMP_MILLIS)) {
        long millisUtc = floorDiv(type.getLong(block, position), MICROSECONDS_PER_MILLISECOND);
        return new Date(millisUtc);
    }
    if (type.equals(TimestampWithTimeZoneType.TIMESTAMP_TZ_MILLIS)) {
        long millisUtc = unpackMillisUtc(type.getLong(block, position));
        return new Date(millisUtc);
    }
    if (type instanceof DecimalType) {
        return readBigDecimal((DecimalType) type, block, position);
    }
    if (isJsonType(type)) {
        String json = type.getSlice(block, position).toStringUtf8();
        try {
            return Document.parse(json);
        } catch (BsonInvalidOperationException e) {
            throw new TrinoException(NOT_SUPPORTED, "Can't convert json to MongoDB Document: " + json, e);
        }
    }
    if (isArrayType(type)) {
        Type elementType = type.getTypeParameters().get(0);
        Block arrayBlock = block.getObject(position, Block.class);
        List<Object> list = new ArrayList<>(arrayBlock.getPositionCount());
        for (int i = 0; i < arrayBlock.getPositionCount(); i++) {
            Object element = getObjectValue(elementType, arrayBlock, i);
            list.add(element);
        }
        return unmodifiableList(list);
    }
    if (isMapType(type)) {
        Type keyType = type.getTypeParameters().get(0);
        Type valueType = type.getTypeParameters().get(1);
        Block mapBlock = block.getObject(position, Block.class);
        // map type is converted into list of fixed keys document
        List<Object> values = new ArrayList<>(mapBlock.getPositionCount() / 2);
        for (int i = 0; i < mapBlock.getPositionCount(); i += 2) {
            Map<String, Object> mapValue = new HashMap<>();
            mapValue.put("key", getObjectValue(keyType, mapBlock, i));
            mapValue.put("value", getObjectValue(valueType, mapBlock, i + 1));
            values.add(mapValue);
        }
        return unmodifiableList(values);
    }
    if (isRowType(type)) {
        Block rowBlock = block.getObject(position, Block.class);
        List<Type> fieldTypes = type.getTypeParameters();
        if (fieldTypes.size() != rowBlock.getPositionCount()) {
            throw new TrinoException(StandardErrorCode.GENERIC_INTERNAL_ERROR, "Expected row value field count does not match type field count");
        }
        if (isImplicitRowType(type)) {
            List<Object> rowValue = new ArrayList<>();
            for (int i = 0; i < rowBlock.getPositionCount(); i++) {
                Object element = getObjectValue(fieldTypes.get(i), rowBlock, i);
                rowValue.add(element);
            }
            return unmodifiableList(rowValue);
        }
        Map<String, Object> rowValue = new HashMap<>();
        for (int i = 0; i < rowBlock.getPositionCount(); i++) {
            rowValue.put(type.getTypeSignature().getParameters().get(i).getNamedTypeSignature().getName().orElse("field" + i), getObjectValue(fieldTypes.get(i), rowBlock, i));
        }
        return unmodifiableMap(rowValue);
    }
    throw new TrinoException(NOT_SUPPORTED, "unsupported type: " + type);
}
Also used : BsonInvalidOperationException(org.bson.BsonInvalidOperationException) ObjectId(org.bson.types.ObjectId) VarcharType(io.trino.spi.type.VarcharType) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Date(java.util.Date) DateType(io.trino.spi.type.DateType) BooleanType(io.trino.spi.type.BooleanType) TimestampWithTimeZoneType(io.trino.spi.type.TimestampWithTimeZoneType) SmallintType(io.trino.spi.type.SmallintType) TypeUtils.isMapType(io.trino.plugin.mongodb.TypeUtils.isMapType) DecimalType(io.trino.spi.type.DecimalType) TypeUtils.isArrayType(io.trino.plugin.mongodb.TypeUtils.isArrayType) TimeType(io.trino.spi.type.TimeType) DoubleType(io.trino.spi.type.DoubleType) Type(io.trino.spi.type.Type) BigintType(io.trino.spi.type.BigintType) VarcharType(io.trino.spi.type.VarcharType) TypeUtils.isJsonType(io.trino.plugin.mongodb.TypeUtils.isJsonType) TinyintType(io.trino.spi.type.TinyintType) IntegerType(io.trino.spi.type.IntegerType) VarbinaryType(io.trino.spi.type.VarbinaryType) CharType(io.trino.spi.type.CharType) RealType(io.trino.spi.type.RealType) TypeUtils.isRowType(io.trino.plugin.mongodb.TypeUtils.isRowType) DecimalType(io.trino.spi.type.DecimalType) TrinoException(io.trino.spi.TrinoException) Block(io.trino.spi.block.Block) CharType(io.trino.spi.type.CharType) Binary(org.bson.types.Binary)

Example 69 with DecimalType

use of io.trino.spi.type.DecimalType in project trino by trinodb.

the class TestOrcFileRewriter method testRewrite.

@Test
public void testRewrite() throws Exception {
    ArrayType arrayType = new ArrayType(BIGINT);
    ArrayType arrayOfArrayType = new ArrayType(arrayType);
    Type mapType = TESTING_TYPE_MANAGER.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.typeParameter(createVarcharType(5).getTypeSignature()), TypeSignatureParameter.typeParameter(BOOLEAN.getTypeSignature())));
    List<Long> columnIds = ImmutableList.of(3L, 7L, 9L, 10L, 11L, 12L);
    DecimalType decimalType = DecimalType.createDecimalType(4, 4);
    List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), arrayType, mapType, arrayOfArrayType, decimalType);
    File file = temporary.resolve(randomUUID().toString()).toFile();
    try (OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file)) {
        List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello", arrayBlockOf(BIGINT, 1, 2), mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)), new BigDecimal("2.3")).row(777L, "sky", arrayBlockOf(BIGINT, 3, 4), mapBlockOf(createVarcharType(5), BOOLEAN, "k2", false), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 6)), new BigDecimal("2.3")).row(456L, "bye", arrayBlockOf(BIGINT, 5, 6), mapBlockOf(createVarcharType(5), BOOLEAN, "k3", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 7)), new BigDecimal("2.3")).row(888L, "world", arrayBlockOf(BIGINT, 7, 8), mapBlockOf(createVarcharType(5), BOOLEAN, "k4", true), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 8), null), new BigDecimal("2.3")).row(999L, "done", arrayBlockOf(BIGINT, 9, 10), mapBlockOf(createVarcharType(5), BOOLEAN, "k5", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 9, 10)), new BigDecimal("2.3")).build();
        writer.appendPages(pages);
    }
    try (OrcDataSource dataSource = fileOrcDataSource(file)) {
        OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes);
        assertEquals(reader.getReaderRowCount(), 5);
        assertEquals(reader.getFileRowCount(), 5);
        assertEquals(reader.getSplitLength(), file.length());
        Page page = reader.nextPage();
        assertEquals(page.getPositionCount(), 5);
        Block column0 = page.getBlock(0);
        assertEquals(column0.getPositionCount(), 5);
        for (int i = 0; i < 5; i++) {
            assertEquals(column0.isNull(i), false);
        }
        assertEquals(BIGINT.getLong(column0, 0), 123L);
        assertEquals(BIGINT.getLong(column0, 1), 777L);
        assertEquals(BIGINT.getLong(column0, 2), 456L);
        assertEquals(BIGINT.getLong(column0, 3), 888L);
        assertEquals(BIGINT.getLong(column0, 4), 999L);
        Block column1 = page.getBlock(1);
        assertEquals(column1.getPositionCount(), 5);
        for (int i = 0; i < 5; i++) {
            assertEquals(column1.isNull(i), false);
        }
        assertEquals(createVarcharType(20).getSlice(column1, 0), utf8Slice("hello"));
        assertEquals(createVarcharType(20).getSlice(column1, 1), utf8Slice("sky"));
        assertEquals(createVarcharType(20).getSlice(column1, 2), utf8Slice("bye"));
        assertEquals(createVarcharType(20).getSlice(column1, 3), utf8Slice("world"));
        assertEquals(createVarcharType(20).getSlice(column1, 4), utf8Slice("done"));
        Block column2 = page.getBlock(2);
        assertEquals(column2.getPositionCount(), 5);
        for (int i = 0; i < 5; i++) {
            assertEquals(column2.isNull(i), false);
        }
        assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 0), arrayBlockOf(BIGINT, 1, 2)));
        assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 1), arrayBlockOf(BIGINT, 3, 4)));
        assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 2), arrayBlockOf(BIGINT, 5, 6)));
        assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 3), arrayBlockOf(BIGINT, 7, 8)));
        assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 4), arrayBlockOf(BIGINT, 9, 10)));
        Block column3 = page.getBlock(3);
        assertEquals(column3.getPositionCount(), 5);
        for (int i = 0; i < 5; i++) {
            assertEquals(column3.isNull(i), false);
        }
        assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, arrayType.getObject(column3, 0), mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true)));
        assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, arrayType.getObject(column3, 1), mapBlockOf(createVarcharType(5), BOOLEAN, "k2", false)));
        assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, arrayType.getObject(column3, 2), mapBlockOf(createVarcharType(5), BOOLEAN, "k3", true)));
        assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, arrayType.getObject(column3, 3), mapBlockOf(createVarcharType(5), BOOLEAN, "k4", true)));
        assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, arrayType.getObject(column3, 4), mapBlockOf(createVarcharType(5), BOOLEAN, "k5", true)));
        Block column4 = page.getBlock(4);
        assertEquals(column4.getPositionCount(), 5);
        for (int i = 0; i < 5; i++) {
            assertEquals(column4.isNull(i), false);
        }
        assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 0), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5))));
        assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 1), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 6))));
        assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 2), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 7))));
        assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 3), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 8), null)));
        assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 4), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 9, 10))));
        assertNull(reader.nextPage());
        OrcFileMetadata orcFileMetadata = METADATA_CODEC.fromJson(reader.getUserMetadata().get(OrcFileMetadata.KEY).getBytes());
        assertEquals(orcFileMetadata, new OrcFileMetadata(ImmutableMap.<Long, TypeId>builder().put(3L, BIGINT.getTypeId()).put(7L, createVarcharType(20).getTypeId()).put(9L, arrayType.getTypeId()).put(10L, mapType.getTypeId()).put(11L, arrayOfArrayType.getTypeId()).put(12L, decimalType.getTypeId()).buildOrThrow()));
    }
    BitSet rowsToDelete = new BitSet(5);
    rowsToDelete.set(1);
    rowsToDelete.set(3);
    rowsToDelete.set(4);
    File newFile = temporary.resolve(randomUUID().toString()).toFile();
    OrcFileInfo info = OrcFileRewriter.rewrite(file, newFile, rowsToDelete);
    assertEquals(info.getRowCount(), 2);
    assertEquals(info.getUncompressedSize(), 94);
    try (OrcDataSource dataSource = fileOrcDataSource(newFile)) {
        OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes);
        assertEquals(reader.getReaderRowCount(), 2);
        assertEquals(reader.getFileRowCount(), 2);
        assertEquals(reader.getSplitLength(), newFile.length());
        Page page = reader.nextPage();
        assertEquals(page.getPositionCount(), 2);
        Block column0 = page.getBlock(0);
        assertEquals(column0.getPositionCount(), 2);
        for (int i = 0; i < 2; i++) {
            assertEquals(column0.isNull(i), false);
        }
        assertEquals(BIGINT.getLong(column0, 0), 123L);
        assertEquals(BIGINT.getLong(column0, 1), 456L);
        Block column1 = page.getBlock(1);
        assertEquals(column1.getPositionCount(), 2);
        for (int i = 0; i < 2; i++) {
            assertEquals(column1.isNull(i), false);
        }
        assertEquals(createVarcharType(20).getSlice(column1, 0), utf8Slice("hello"));
        assertEquals(createVarcharType(20).getSlice(column1, 1), utf8Slice("bye"));
        Block column2 = page.getBlock(2);
        assertEquals(column2.getPositionCount(), 2);
        for (int i = 0; i < 2; i++) {
            assertEquals(column2.isNull(i), false);
        }
        assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 0), arrayBlockOf(BIGINT, 1, 2)));
        assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 1), arrayBlockOf(BIGINT, 5, 6)));
        Block column3 = page.getBlock(3);
        assertEquals(column3.getPositionCount(), 2);
        for (int i = 0; i < 2; i++) {
            assertEquals(column3.isNull(i), false);
        }
        assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, arrayType.getObject(column3, 0), mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true)));
        assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, arrayType.getObject(column3, 1), mapBlockOf(createVarcharType(5), BOOLEAN, "k3", true)));
        Block column4 = page.getBlock(4);
        assertEquals(column4.getPositionCount(), 2);
        for (int i = 0; i < 2; i++) {
            assertEquals(column4.isNull(i), false);
        }
        assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 0), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5))));
        assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column4, 1), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 7))));
        assertEquals(reader.nextPage(), null);
        OrcFileMetadata orcFileMetadata = METADATA_CODEC.fromJson(reader.getUserMetadata().get(OrcFileMetadata.KEY).getBytes());
        assertEquals(orcFileMetadata, new OrcFileMetadata(ImmutableMap.<Long, TypeId>builder().put(3L, BIGINT.getTypeId()).put(7L, createVarcharType(20).getTypeId()).put(9L, arrayType.getTypeId()).put(10L, mapType.getTypeId()).put(11L, arrayOfArrayType.getTypeId()).put(12L, decimalType.getTypeId()).buildOrThrow()));
    }
}
Also used : OrcTestingUtil.fileOrcDataSource(io.trino.plugin.raptor.legacy.storage.OrcTestingUtil.fileOrcDataSource) OrcDataSource(io.trino.orc.OrcDataSource) BitSet(java.util.BitSet) Page(io.trino.spi.Page) OrcRecordReader(io.trino.orc.OrcRecordReader) BigDecimal(java.math.BigDecimal) ArrayType(io.trino.spi.type.ArrayType) Type(io.trino.spi.type.Type) ArrayType(io.trino.spi.type.ArrayType) VarcharType.createVarcharType(io.trino.spi.type.VarcharType.createVarcharType) DecimalType(io.trino.spi.type.DecimalType) OrcFileInfo(io.trino.plugin.raptor.legacy.storage.OrcFileRewriter.OrcFileInfo) DecimalType(io.trino.spi.type.DecimalType) Block(io.trino.spi.block.Block) File(java.io.File) Test(org.testng.annotations.Test)

Example 70 with DecimalType

use of io.trino.spi.type.DecimalType in project trino by trinodb.

the class KuduPageSink method appendColumn.

private void appendColumn(PartialRow row, Page page, int position, int channel, int destChannel) {
    Block block = page.getBlock(channel);
    Type type = columnTypes.get(destChannel);
    if (block.isNull(position)) {
        row.setNull(destChannel);
    } else if (TIMESTAMP_MILLIS.equals(type)) {
        row.addLong(destChannel, truncateEpochMicrosToMillis(type.getLong(block, position)));
    } else if (REAL.equals(type)) {
        row.addFloat(destChannel, intBitsToFloat(toIntExact(type.getLong(block, position))));
    } else if (BIGINT.equals(type)) {
        row.addLong(destChannel, type.getLong(block, position));
    } else if (INTEGER.equals(type)) {
        row.addInt(destChannel, toIntExact(type.getLong(block, position)));
    } else if (SMALLINT.equals(type)) {
        row.addShort(destChannel, Shorts.checkedCast(type.getLong(block, position)));
    } else if (TINYINT.equals(type)) {
        row.addByte(destChannel, SignedBytes.checkedCast(type.getLong(block, position)));
    } else if (BOOLEAN.equals(type)) {
        row.addBoolean(destChannel, type.getBoolean(block, position));
    } else if (DOUBLE.equals(type)) {
        row.addDouble(destChannel, type.getDouble(block, position));
    } else if (type instanceof VarcharType) {
        Type originalType = originalColumnTypes.get(destChannel);
        if (DATE.equals(originalType)) {
            SqlDate date = (SqlDate) originalType.getObjectValue(connectorSession, block, position);
            LocalDateTime ldt = LocalDateTime.ofEpochSecond(TimeUnit.DAYS.toSeconds(date.getDays()), 0, ZoneOffset.UTC);
            byte[] bytes = ldt.format(DateTimeFormatter.ISO_LOCAL_DATE).getBytes(StandardCharsets.UTF_8);
            row.addStringUtf8(destChannel, bytes);
        } else {
            row.addString(destChannel, type.getSlice(block, position).toStringUtf8());
        }
    } else if (VARBINARY.equals(type)) {
        row.addBinary(destChannel, type.getSlice(block, position).toByteBuffer());
    } else if (type instanceof DecimalType) {
        SqlDecimal sqlDecimal = (SqlDecimal) type.getObjectValue(connectorSession, block, position);
        row.addDecimal(destChannel, sqlDecimal.toBigDecimal());
    } else {
        throw new UnsupportedOperationException("Type is not supported: " + type);
    }
}
Also used : LocalDateTime(java.time.LocalDateTime) Type(io.trino.spi.type.Type) VarcharType(io.trino.spi.type.VarcharType) DecimalType(io.trino.spi.type.DecimalType) VarcharType(io.trino.spi.type.VarcharType) SqlDate(io.trino.spi.type.SqlDate) Block(io.trino.spi.block.Block) DecimalType(io.trino.spi.type.DecimalType) SqlDecimal(io.trino.spi.type.SqlDecimal)

Aggregations

DecimalType (io.trino.spi.type.DecimalType)79 VarcharType (io.trino.spi.type.VarcharType)50 CharType (io.trino.spi.type.CharType)39 TrinoException (io.trino.spi.TrinoException)31 Type (io.trino.spi.type.Type)29 TimestampType (io.trino.spi.type.TimestampType)23 DecimalType.createDecimalType (io.trino.spi.type.DecimalType.createDecimalType)22 ArrayType (io.trino.spi.type.ArrayType)21 BigDecimal (java.math.BigDecimal)19 Int128 (io.trino.spi.type.Int128)16 BigInteger (java.math.BigInteger)15 Block (io.trino.spi.block.Block)14 Slice (io.airlift.slice.Slice)13 TimeType (io.trino.spi.type.TimeType)13 TimestampWithTimeZoneType (io.trino.spi.type.TimestampWithTimeZoneType)13 VarcharType.createUnboundedVarcharType (io.trino.spi.type.VarcharType.createUnboundedVarcharType)13 MapType (io.trino.spi.type.MapType)12 ArrayList (java.util.ArrayList)12 ImmutableList (com.google.common.collect.ImmutableList)11 RowType (io.trino.spi.type.RowType)11