Examples with RowType io.trino.spi.type.RowType used on opensource projects

Example 26 with RowType

use of io.trino.spi.type.RowType in project trino by trinodb.

the class OrcTester method testStructRoundTrip.

private void testStructRoundTrip(Type type, List<?> values) throws Exception {
    Type rowType = rowType(type, type, type);
    // values in simple struct
    testRoundTripType(rowType, values.stream().map(OrcTester::toHiveStruct).collect(toList()));
    if (structuralNullTestsEnabled) {
        // values and nulls in simple struct
        testRoundTripType(rowType, insertNullEvery(5, values).stream().map(OrcTester::toHiveStruct).collect(toList()));
        // all null values in simple struct
        testRoundTripType(rowType, values.stream().map(value -> toHiveStruct(null)).collect(toList()));
    }
    if (missingStructFieldsTestsEnabled) {
        Type readType = rowType(type, type, type, type, type, type);
        Type writeType = rowType(type, type, type);
        List<?> writeValues = values.stream().map(OrcTester::toHiveStruct).collect(toList());
        List<?> readValues = values.stream().map(OrcTester::toHiveStructWithNull).collect(toList());
        assertRoundTrip(writeType, readType, writeValues, readValues);
    }
}

Example 27 with RowType

use of io.trino.spi.type.RowType in project trino by trinodb.

the class TestOrcWithoutRowGroupInfo method testAndVerifyResults.

private void testAndVerifyResults(OrcPredicate orcPredicate) throws Exception {
    // this file was written by minor compaction in hive
    File file = new File(getResource("orcFileWithoutRowGroupInfo.orc").toURI());
    OrcReader orcReader = createOrcReader(new FileOrcDataSource(file, new OrcReaderOptions()), new OrcReaderOptions()).orElseThrow();
    assertEquals(orcReader.getFooter().getNumberOfRows(), 2);
    assertEquals(orcReader.getFooter().getRowsInRowGroup(), OptionalInt.empty());
    RowType rowType = RowType.from(ImmutableList.of(RowType.field("a", BIGINT)));
    OrcRecordReader reader = orcReader.createRecordReader(orcReader.getRootColumn().getNestedColumns(), ImmutableList.of(INTEGER, BIGINT, INTEGER, BIGINT, BIGINT, rowType), orcPredicate, DateTimeZone.UTC, newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE, RuntimeException::new);
    int rows = 0;
    while (true) {
        Page page = reader.nextPage();
        if (page == null) {
            break;
        }
        page = page.getLoadedPage();
        rows += page.getPositionCount();
        Block rowBlock = page.getBlock(5);
        for (int position = 0; position < page.getPositionCount(); position++) {
            BIGINT.getLong(rowType.getObject(rowBlock, 0), 0);
        }
    }
    assertEquals(rows, reader.getFileRowCount());
}

Example 28 with RowType

use of io.trino.spi.type.RowType in project trino by trinodb.

the class RcFileTester method preprocessWriteValueOld.

private static Object preprocessWriteValueOld(Format format, Type type, Object value) {
    if (value == null) {
        return null;
    }
    if (type.equals(BOOLEAN)) {
        return value;
    }
    if (type.equals(TINYINT)) {
        return ((Number) value).byteValue();
    }
    if (type.equals(SMALLINT)) {
        return ((Number) value).shortValue();
    }
    if (type.equals(INTEGER)) {
        return ((Number) value).intValue();
    }
    if (type.equals(BIGINT)) {
        return ((Number) value).longValue();
    }
    if (type.equals(REAL)) {
        return ((Number) value).floatValue();
    }
    if (type.equals(DOUBLE)) {
        return ((Number) value).doubleValue();
    }
    if (type instanceof VarcharType) {
        return value;
    }
    if (type.equals(VARBINARY)) {
        return ((SqlVarbinary) value).getBytes();
    }
    if (type.equals(DATE)) {
        return Date.ofEpochDay(((SqlDate) value).getDays());
    }
    if (type.equals(TIMESTAMP_MILLIS)) {
        long millis = ((SqlTimestamp) value).getMillis();
        if (format == Format.BINARY) {
            millis = HIVE_STORAGE_TIME_ZONE.convertLocalToUTC(millis, false);
        }
        return Timestamp.ofEpochMilli(millis);
    }
    if (type instanceof DecimalType) {
        return HiveDecimal.create(((SqlDecimal) value).toBigDecimal());
    }
    if (type instanceof ArrayType) {
        Type elementType = type.getTypeParameters().get(0);
        return ((List<?>) value).stream().map(element -> preprocessWriteValueOld(format, elementType, element)).collect(toList());
    }
    if (type instanceof MapType) {
        Type keyType = type.getTypeParameters().get(0);
        Type valueType = type.getTypeParameters().get(1);
        Map<Object, Object> newMap = new HashMap<>();
        for (Entry<?, ?> entry : ((Map<?, ?>) value).entrySet()) {
            newMap.put(preprocessWriteValueOld(format, keyType, entry.getKey()), preprocessWriteValueOld(format, valueType, entry.getValue()));
        }
        return newMap;
    }
    if (type instanceof RowType) {
        List<?> fieldValues = (List<?>) value;
        List<Type> fieldTypes = type.getTypeParameters();
        List<Object> newStruct = new ArrayList<>();
        for (int fieldId = 0; fieldId < fieldValues.size(); fieldId++) {
            newStruct.add(preprocessWriteValueOld(format, fieldTypes.get(fieldId), fieldValues.get(fieldId)));
        }
        return newStruct;
    }
    throw new IllegalArgumentException("unsupported type: " + type);
}

Example 29 with RowType

use of io.trino.spi.type.RowType in project trino by trinodb.

the class RcFileTester method writeValue.

private static void writeValue(Type type, BlockBuilder blockBuilder, Object value) {
    if (value == null) {
        blockBuilder.appendNull();
    } else {
        if (BOOLEAN.equals(type)) {
            type.writeBoolean(blockBuilder, (Boolean) value);
        } else if (TINYINT.equals(type)) {
            type.writeLong(blockBuilder, ((Number) value).longValue());
        } else if (SMALLINT.equals(type)) {
            type.writeLong(blockBuilder, ((Number) value).longValue());
        } else if (INTEGER.equals(type)) {
            type.writeLong(blockBuilder, ((Number) value).longValue());
        } else if (BIGINT.equals(type)) {
            type.writeLong(blockBuilder, ((Number) value).longValue());
        } else if (Decimals.isShortDecimal(type)) {
            type.writeLong(blockBuilder, ((SqlDecimal) value).toBigDecimal().unscaledValue().longValue());
        } else if (Decimals.isLongDecimal(type)) {
            type.writeObject(blockBuilder, Int128.valueOf(((SqlDecimal) value).toBigDecimal().unscaledValue()));
        } else if (REAL.equals(type)) {
            type.writeLong(blockBuilder, Float.floatToIntBits((Float) value));
        } else if (DOUBLE.equals(type)) {
            type.writeDouble(blockBuilder, ((Number) value).doubleValue());
        } else if (VARCHAR.equals(type)) {
            type.writeSlice(blockBuilder, Slices.utf8Slice((String) value));
        } else if (VARBINARY.equals(type)) {
            type.writeSlice(blockBuilder, Slices.wrappedBuffer(((SqlVarbinary) value).getBytes()));
        } else if (DATE.equals(type)) {
            long days = ((SqlDate) value).getDays();
            type.writeLong(blockBuilder, days);
        } else if (TIMESTAMP_MILLIS.equals(type)) {
            long millis = ((SqlTimestamp) value).getMillis();
            type.writeLong(blockBuilder, millis * MICROSECONDS_PER_MILLISECOND);
        } else {
            if (type instanceof ArrayType) {
                List<?> array = (List<?>) value;
                Type elementType = type.getTypeParameters().get(0);
                BlockBuilder arrayBlockBuilder = blockBuilder.beginBlockEntry();
                for (Object elementValue : array) {
                    writeValue(elementType, arrayBlockBuilder, elementValue);
                }
                blockBuilder.closeEntry();
            } else if (type instanceof MapType) {
                Map<?, ?> map = (Map<?, ?>) value;
                Type keyType = type.getTypeParameters().get(0);
                Type valueType = type.getTypeParameters().get(1);
                BlockBuilder mapBlockBuilder = blockBuilder.beginBlockEntry();
                for (Entry<?, ?> entry : map.entrySet()) {
                    writeValue(keyType, mapBlockBuilder, entry.getKey());
                    writeValue(valueType, mapBlockBuilder, entry.getValue());
                }
                blockBuilder.closeEntry();
            } else if (type instanceof RowType) {
                List<?> array = (List<?>) value;
                List<Type> fieldTypes = type.getTypeParameters();
                BlockBuilder rowBlockBuilder = blockBuilder.beginBlockEntry();
                for (int fieldId = 0; fieldId < fieldTypes.size(); fieldId++) {
                    Type fieldType = fieldTypes.get(fieldId);
                    writeValue(fieldType, rowBlockBuilder, array.get(fieldId));
                }
                blockBuilder.closeEntry();
            } else {
                throw new IllegalArgumentException("Unsupported type " + type);
            }
        }
    }
}

Example 30 with RowType

use of io.trino.spi.type.RowType in project trino by trinodb.

the class CheckpointSchemaManager method getAddEntryType.

public RowType getAddEntryType(MetadataEntry metadataEntry) {
    List<ColumnMetadata> allColumns = extractSchema(metadataEntry, typeManager);
    List<ColumnMetadata> minMaxColumns = columnsWithStats(metadataEntry, typeManager);
    ImmutableList.Builder<RowType.Field> minMaxFields = ImmutableList.builder();
    for (ColumnMetadata dataColumn : minMaxColumns) {
        Type type = dataColumn.getType();
        if (type instanceof TimestampWithTimeZoneType) {
            minMaxFields.add(RowType.field(dataColumn.getName(), TimestampType.TIMESTAMP_MILLIS));
        } else {
            minMaxFields.add(RowType.field(dataColumn.getName(), type));
        }
    }
    ImmutableList.Builder<RowType.Field> statsColumns = ImmutableList.builder();
    statsColumns.add(RowType.field("numRecords", BigintType.BIGINT));
    List<RowType.Field> minMax = minMaxFields.build();
    if (!minMax.isEmpty()) {
        RowType minMaxType = RowType.from(minMax);
        statsColumns.add(RowType.field("minValues", minMaxType));
        statsColumns.add(RowType.field("maxValues", minMaxType));
    }
    statsColumns.add(RowType.field("nullCount", RowType.from(allColumns.stream().map(column -> buildNullCountType(Optional.of(column.getName()), column.getType())).collect(toImmutableList()))));
    MapType stringMap = (MapType) typeManager.getType(TypeSignature.mapType(VarcharType.VARCHAR.getTypeSignature(), VarcharType.VARCHAR.getTypeSignature()));
    List<RowType.Field> addFields = ImmutableList.of(RowType.field("path", VarcharType.createUnboundedVarcharType()), RowType.field("partitionValues", stringMap), RowType.field("size", BigintType.BIGINT), RowType.field("modificationTime", BigintType.BIGINT), RowType.field("dataChange", BooleanType.BOOLEAN), RowType.field("stats", VarcharType.createUnboundedVarcharType()), RowType.field("stats_parsed", RowType.from(statsColumns.build())), RowType.field("tags", stringMap));
    return RowType.from(addFields);
}