Examples with ColumnMetadata io.trino.spi.connector.ColumnMetadata used on opensource projects

Example 46 with ColumnMetadata

use of io.trino.spi.connector.ColumnMetadata in project trino by trinodb.

the class CheckpointSchemaManager method getAddEntryType.

public RowType getAddEntryType(MetadataEntry metadataEntry) {
    List<ColumnMetadata> allColumns = extractSchema(metadataEntry, typeManager);
    List<ColumnMetadata> minMaxColumns = columnsWithStats(metadataEntry, typeManager);
    ImmutableList.Builder<RowType.Field> minMaxFields = ImmutableList.builder();
    for (ColumnMetadata dataColumn : minMaxColumns) {
        Type type = dataColumn.getType();
        if (type instanceof TimestampWithTimeZoneType) {
            minMaxFields.add(RowType.field(dataColumn.getName(), TimestampType.TIMESTAMP_MILLIS));
        } else {
            minMaxFields.add(RowType.field(dataColumn.getName(), type));
        }
    }
    ImmutableList.Builder<RowType.Field> statsColumns = ImmutableList.builder();
    statsColumns.add(RowType.field("numRecords", BigintType.BIGINT));
    List<RowType.Field> minMax = minMaxFields.build();
    if (!minMax.isEmpty()) {
        RowType minMaxType = RowType.from(minMax);
        statsColumns.add(RowType.field("minValues", minMaxType));
        statsColumns.add(RowType.field("maxValues", minMaxType));
    }
    statsColumns.add(RowType.field("nullCount", RowType.from(allColumns.stream().map(column -> buildNullCountType(Optional.of(column.getName()), column.getType())).collect(toImmutableList()))));
    MapType stringMap = (MapType) typeManager.getType(TypeSignature.mapType(VarcharType.VARCHAR.getTypeSignature(), VarcharType.VARCHAR.getTypeSignature()));
    List<RowType.Field> addFields = ImmutableList.of(RowType.field("path", VarcharType.createUnboundedVarcharType()), RowType.field("partitionValues", stringMap), RowType.field("size", BigintType.BIGINT), RowType.field("modificationTime", BigintType.BIGINT), RowType.field("dataChange", BooleanType.BOOLEAN), RowType.field("stats", VarcharType.createUnboundedVarcharType()), RowType.field("stats_parsed", RowType.from(statsColumns.build())), RowType.field("tags", stringMap));
    return RowType.from(addFields);
}

Example 47 with ColumnMetadata

use of io.trino.spi.connector.ColumnMetadata in project trino by trinodb.

the class CheckpointEntryIterator method readNullCount.

private Map<String, Object> readNullCount(Block block, int blockPosition, List<ColumnMetadata> columns) {
    if (block.isNull(blockPosition)) {
        // Statistics were not collected
        return ImmutableMap.of();
    }
    Block valuesBlock = block.getObject(blockPosition, Block.class);
    ImmutableMap.Builder<String, Object> values = ImmutableMap.builder();
    for (int i = 0; i < columns.size(); i++) {
        ColumnMetadata metadata = columns.get(i);
        if (valuesBlock.isNull(i)) {
            continue;
        }
        if (metadata.getType() instanceof RowType) {
            if (checkpointRowStatisticsWritingEnabled) {
                // RowType column statistics are not used for query planning, but need to be copied when writing out new Checkpoint files.
                values.put(metadata.getName(), valuesBlock.getSingleValueBlock(i));
            }
            continue;
        }
        values.put(metadata.getName(), getLong(valuesBlock, i));
    }
    return values.buildOrThrow();
}

Example 48 with ColumnMetadata

use of io.trino.spi.connector.ColumnMetadata in project trino by trinodb.

the class CheckpointEntryIterator method readMinMax.

private Map<String, Object> readMinMax(Block block, int blockPosition, List<ColumnMetadata> eligibleColumns) {
    if (block.isNull(blockPosition)) {
        // Statistics were not collected
        return ImmutableMap.of();
    }
    Block valuesBlock = block.getObject(blockPosition, Block.class);
    ImmutableMap.Builder<String, Object> values = ImmutableMap.builder();
    for (int i = 0; i < eligibleColumns.size(); i++) {
        ColumnMetadata metadata = eligibleColumns.get(i);
        String name = metadata.getName();
        Type type = metadata.getType();
        if (valuesBlock.isNull(i)) {
            continue;
        }
        if (type instanceof RowType) {
            if (checkpointRowStatisticsWritingEnabled) {
                // RowType column statistics are not used for query planning, but need to be copied when writing out new Checkpoint files.
                values.put(name, valuesBlock.getSingleValueBlock(i));
            }
            continue;
        }
        if (type instanceof TimestampWithTimeZoneType) {
            Instant instant = Instant.ofEpochMilli(LongMath.divide((Long) readNativeValue(TIMESTAMP_MILLIS, valuesBlock, i), MICROSECONDS_PER_MILLISECOND, UNNECESSARY));
            if (!instant.atZone(UTC).toLocalDate().isBefore(START_OF_MODERN_ERA)) {
                values.put(name, packDateTimeWithZone(instant.toEpochMilli(), UTC_KEY));
            }
            continue;
        }
        values.put(name, readNativeValue(type, valuesBlock, i));
    }
    return values.buildOrThrow();
}

Example 49 with ColumnMetadata

use of io.trino.spi.connector.ColumnMetadata in project trino by trinodb.

the class TestPruneTableScanColumns method testPushColumnPruningProjection.

@Test
public void testPushColumnPruningProjection() {
    try (RuleTester ruleTester = defaultRuleTester()) {
        String mockCatalog = "mock_catalog";
        String testSchema = "test_schema";
        String testTable = "test_table";
        SchemaTableName testSchemaTable = new SchemaTableName(testSchema, testTable);
        ColumnHandle columnHandleA = new MockConnectorColumnHandle("cola", DATE);
        ColumnHandle columnHandleB = new MockConnectorColumnHandle("colb", DOUBLE);
        Map<String, ColumnHandle> assignments = ImmutableMap.of("cola", columnHandleA, "colb", columnHandleB);
        // Create catalog with applyProjection
        MockConnectorFactory factory = MockConnectorFactory.builder().withListSchemaNames(connectorSession -> ImmutableList.of(testSchema)).withListTables((connectorSession, schema) -> testSchema.equals(schema) ? ImmutableList.of(testSchemaTable) : ImmutableList.of()).withGetColumns(schemaTableName -> assignments.entrySet().stream().map(entry -> new ColumnMetadata(entry.getKey(), ((MockConnectorColumnHandle) entry.getValue()).getType())).collect(toImmutableList())).withApplyProjection(this::mockApplyProjection).build();
        ruleTester.getQueryRunner().createCatalog(mockCatalog, factory, ImmutableMap.of());
        ruleTester.assertThat(new PruneTableScanColumns(ruleTester.getMetadata())).on(p -> {
            Symbol symbolA = p.symbol("cola", DATE);
            Symbol symbolB = p.symbol("colb", DOUBLE);
            return p.project(Assignments.of(p.symbol("x"), symbolB.toSymbolReference()), p.tableScan(new TableHandle(new CatalogName(mockCatalog), new MockConnectorTableHandle(testSchemaTable), MockConnectorTransactionHandle.INSTANCE), ImmutableList.of(symbolA, symbolB), ImmutableMap.of(symbolA, columnHandleA, symbolB, columnHandleB)));
        }).withSession(testSessionBuilder().setCatalog(mockCatalog).setSchema(testSchema).build()).matches(strictProject(ImmutableMap.of("expr", PlanMatchPattern.expression("COLB")), tableScan(new MockConnectorTableHandle(testSchemaTable, TupleDomain.all(), Optional.of(ImmutableList.of(columnHandleB)))::equals, TupleDomain.all(), ImmutableMap.of("COLB", columnHandleB::equals))));
    }
}

Example 50 with ColumnMetadata

use of io.trino.spi.connector.ColumnMetadata in project trino by trinodb.

the class TestPartialTopNWithPresortedInput method createLocalQueryRunner.

@Override
protected LocalQueryRunner createLocalQueryRunner() {
    Session session = testSessionBuilder().setCatalog(MOCK_CATALOG).setSchema(TEST_SCHEMA).build();
    LocalQueryRunner queryRunner = LocalQueryRunner.builder(session).build();
    MockConnectorFactory mockFactory = MockConnectorFactory.builder().withGetTableProperties((connectorSession, handle) -> {
        MockConnectorTableHandle tableHandle = (MockConnectorTableHandle) handle;
        if (tableHandle.getTableName().equals(tableA)) {
            return new ConnectorTableProperties(TupleDomain.all(), Optional.empty(), Optional.empty(), Optional.empty(), ImmutableList.of(new SortingProperty<>(columnHandleA, ASC_NULLS_FIRST)));
        }
        throw new IllegalArgumentException();
    }).withGetColumns(schemaTableName -> {
        if (schemaTableName.equals(tableA)) {
            return ImmutableList.of(new ColumnMetadata(columnNameA, VARCHAR), new ColumnMetadata(columnNameB, VARCHAR));
        }
        throw new IllegalArgumentException();
    }).build();
    queryRunner.createCatalog(MOCK_CATALOG, mockFactory, ImmutableMap.of());
    return queryRunner;
}