Examples with RowType io.trino.spi.type.RowType used on opensource projects

Example 31 with RowType

use of io.trino.spi.type.RowType in project trino by trinodb.

the class CheckpointWriter method writeObjectMapAsFields.

private void writeObjectMapAsFields(BlockBuilder blockBuilder, RowType type, int fieldId, String fieldName, Optional<Map<String, Object>> values) {
    RowType.Field valuesField = validateAndGetField(type, fieldId, fieldName);
    RowType valuesFieldType = (RowType) valuesField.getType();
    BlockBuilder fieldBlockBuilder = blockBuilder.beginBlockEntry();
    if (values.isEmpty()) {
        blockBuilder.appendNull();
    } else {
        for (RowType.Field valueField : valuesFieldType.getFields()) {
            // anonymous row fields are not expected here
            Object value = values.get().get(valueField.getName().orElseThrow());
            if (valueField.getType() instanceof RowType) {
                Block rowBlock = (Block) value;
                checkState(rowBlock.getPositionCount() == 1, "Invalid RowType statistics for writing Delta Lake checkpoint");
                if (rowBlock.isNull(0)) {
                    fieldBlockBuilder.appendNull();
                } else {
                    valueField.getType().appendTo(rowBlock, 0, fieldBlockBuilder);
                }
            } else {
                writeNativeValue(valueField.getType(), fieldBlockBuilder, value);
            }
        }
    }
    blockBuilder.closeEntry();
}

Example 32 with RowType

use of io.trino.spi.type.RowType in project trino by trinodb.

the class CheckpointWriter method writeStringList.

private void writeStringList(BlockBuilder blockBuilder, RowType type, int fieldId, String fieldName, @Nullable List<String> values) {
    RowType.Field field = validateAndGetField(type, fieldId, fieldName);
    checkArgument(field.getType() instanceof ArrayType, "Expected field %s/%s to by of ArrayType but got %s", fieldId, fieldName, field.getType());
    if (values == null) {
        blockBuilder.appendNull();
        return;
    }
    ArrayType arrayType = (ArrayType) field.getType();
    BlockBuilder mapBuilder = blockBuilder.beginBlockEntry();
    for (String value : values) {
        if (value == null) {
            mapBuilder.appendNull();
        } else {
            arrayType.getElementType().writeSlice(mapBuilder, utf8Slice(value));
        }
    }
    blockBuilder.closeEntry();
}

Example 33 with RowType

use of io.trino.spi.type.RowType in project trino by trinodb.

the class CheckpointWriter method write.

public void write(ConnectorSession session, CheckpointEntries entries, Path targetPath) {
    RowType metadataEntryType = checkpointSchemaManager.getMetadataEntryType();
    RowType protocolEntryType = checkpointSchemaManager.getProtocolEntryType();
    RowType txnEntryType = checkpointSchemaManager.getTxnEntryType();
    RowType addEntryType = checkpointSchemaManager.getAddEntryType(entries.getMetadataEntry());
    RowType removeEntryType = checkpointSchemaManager.getRemoveEntryType();
    List<String> columnNames = ImmutableList.of("metaData", "protocol", "txn", "add", "remove");
    List<Type> columnTypes = ImmutableList.of(metadataEntryType, protocolEntryType, txnEntryType, addEntryType, removeEntryType);
    Properties schema = buildSchemaProperties(columnNames, columnTypes);
    Configuration conf = hdfsEnvironment.getConfiguration(new HdfsEnvironment.HdfsContext(session), targetPath);
    configureCompression(conf, SNAPPY);
    JobConf jobConf = toJobConf(conf);
    RecordFileWriter writer = new RecordFileWriter(targetPath, columnNames, fromHiveStorageFormat(PARQUET), schema, PARQUET.getEstimatedWriterMemoryUsage(), jobConf, typeManager, DateTimeZone.UTC, session);
    PageBuilder pageBuilder = new PageBuilder(columnTypes);
    writeMetadataEntry(pageBuilder, metadataEntryType, entries.getMetadataEntry());
    writeProtocolEntry(pageBuilder, protocolEntryType, entries.getProtocolEntry());
    for (TransactionEntry transactionEntry : entries.getTransactionEntries()) {
        writeTransactionEntry(pageBuilder, txnEntryType, transactionEntry);
    }
    for (AddFileEntry addFileEntry : entries.getAddFileEntries()) {
        writeAddFileEntry(pageBuilder, addEntryType, addFileEntry);
    }
    for (RemoveFileEntry removeFileEntry : entries.getRemoveFileEntries()) {
        writeRemoveFileEntry(pageBuilder, removeEntryType, removeFileEntry);
    }
    // Not writing commit infos for now. DB does not keep them in the checkpoints by default
    writer.appendRows(pageBuilder.build());
    writer.commit();
}

Example 34 with RowType

use of io.trino.spi.type.RowType in project trino by trinodb.

the class CheckpointWriter method writeMetadataEntry.

private void writeMetadataEntry(PageBuilder pageBuilder, RowType entryType, MetadataEntry metadataEntry) {
    pageBuilder.declarePosition();
    BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(METADATA_BLOCK_CHANNEL);
    BlockBuilder entryBlockBuilder = blockBuilder.beginBlockEntry();
    writeString(entryBlockBuilder, entryType, 0, "id", metadataEntry.getId());
    writeString(entryBlockBuilder, entryType, 1, "name", metadataEntry.getName());
    writeString(entryBlockBuilder, entryType, 2, "description", metadataEntry.getDescription());
    RowType formatType = getInternalRowType(entryType, 3, "format");
    BlockBuilder formatBlockBuilder = entryBlockBuilder.beginBlockEntry();
    writeString(formatBlockBuilder, formatType, 0, "provider", metadataEntry.getFormat().getProvider());
    writeStringMap(formatBlockBuilder, formatType, 1, "options", metadataEntry.getFormat().getOptions());
    entryBlockBuilder.closeEntry();
    writeString(entryBlockBuilder, entryType, 4, "schemaString", metadataEntry.getSchemaString());
    writeStringList(entryBlockBuilder, entryType, 5, "partitionColumns", metadataEntry.getOriginalPartitionColumns());
    writeStringMap(entryBlockBuilder, entryType, 6, "configuration", metadataEntry.getConfiguration());
    writeLong(entryBlockBuilder, entryType, 7, "createdTime", metadataEntry.getCreatedTime());
    blockBuilder.closeEntry();
    // null for others
    appendNullOtherBlocks(pageBuilder, METADATA_BLOCK_CHANNEL);
}

Example 35 with RowType

use of io.trino.spi.type.RowType in project trino by trinodb.

the class CheckpointEntryIterator method readNullCount.

private Map<String, Object> readNullCount(Block block, int blockPosition, List<ColumnMetadata> columns) {
    if (block.isNull(blockPosition)) {
        // Statistics were not collected
        return ImmutableMap.of();
    }
    Block valuesBlock = block.getObject(blockPosition, Block.class);
    ImmutableMap.Builder<String, Object> values = ImmutableMap.builder();
    for (int i = 0; i < columns.size(); i++) {
        ColumnMetadata metadata = columns.get(i);
        if (valuesBlock.isNull(i)) {
            continue;
        }
        if (metadata.getType() instanceof RowType) {
            if (checkpointRowStatisticsWritingEnabled) {
                // RowType column statistics are not used for query planning, but need to be copied when writing out new Checkpoint files.
                values.put(metadata.getName(), valuesBlock.getSingleValueBlock(i));
            }
            continue;
        }
        values.put(metadata.getName(), getLong(valuesBlock, i));
    }
    return values.buildOrThrow();
}