Examples with TableStatisticsMetadata io.trino.spi.statistics.TableStatisticsMetadata used on opensource projects

Example 1 with TableStatisticsMetadata

use of io.trino.spi.statistics.TableStatisticsMetadata in project trino by trinodb.

the class LogicalPlanner method getInsertPlan.

private RelationPlan getInsertPlan(Analysis analysis, Table table, Query query, TableHandle tableHandle, List<ColumnHandle> insertColumns, Optional<TableLayout> newTableLayout, Optional<WriterTarget> materializedViewRefreshWriterTarget) {
    TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle);
    Map<NodeRef<LambdaArgumentDeclaration>, Symbol> lambdaDeclarationToSymbolMap = buildLambdaDeclarationToSymbolMap(analysis, symbolAllocator);
    RelationPlanner planner = new RelationPlanner(analysis, symbolAllocator, idAllocator, lambdaDeclarationToSymbolMap, plannerContext, Optional.empty(), session, ImmutableMap.of());
    RelationPlan plan = planner.process(query, null);
    ImmutableList.Builder<Symbol> builder = ImmutableList.builder();
    for (int i = 0; i < plan.getFieldMappings().size(); i++) {
        if (!plan.getDescriptor().getFieldByIndex(i).isHidden()) {
            builder.add(plan.getFieldMappings().get(i));
        }
    }
    List<Symbol> visibleFieldMappings = builder.build();
    Map<String, ColumnHandle> columns = metadata.getColumnHandles(session, tableHandle);
    Assignments.Builder assignments = Assignments.builder();
    boolean supportsMissingColumnsOnInsert = metadata.supportsMissingColumnsOnInsert(session, tableHandle);
    ImmutableList.Builder<ColumnMetadata> insertedColumnsBuilder = ImmutableList.builder();
    for (ColumnMetadata column : tableMetadata.getColumns()) {
        if (column.isHidden()) {
            continue;
        }
        Symbol output = symbolAllocator.newSymbol(column.getName(), column.getType());
        int index = insertColumns.indexOf(columns.get(column.getName()));
        if (index < 0) {
            if (supportsMissingColumnsOnInsert) {
                continue;
            }
            Expression cast = new Cast(new NullLiteral(), toSqlType(column.getType()));
            assignments.put(output, cast);
            insertedColumnsBuilder.add(column);
        } else {
            Symbol input = visibleFieldMappings.get(index);
            Type tableType = column.getType();
            Type queryType = symbolAllocator.getTypes().get(input);
            if (queryType.equals(tableType) || typeCoercion.isTypeOnlyCoercion(queryType, tableType)) {
                assignments.put(output, input.toSymbolReference());
            } else {
                Expression cast = noTruncationCast(input.toSymbolReference(), queryType, tableType);
                assignments.put(output, cast);
            }
            insertedColumnsBuilder.add(column);
        }
    }
    ProjectNode projectNode = new ProjectNode(idAllocator.getNextId(), plan.getRoot(), assignments.build());
    List<ColumnMetadata> insertedColumns = insertedColumnsBuilder.build();
    List<Field> fields = insertedColumns.stream().map(column -> Field.newUnqualified(column.getName(), column.getType())).collect(toImmutableList());
    Scope scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(fields)).build();
    plan = new RelationPlan(projectNode, scope, projectNode.getOutputSymbols(), Optional.empty());
    plan = planner.addRowFilters(table, plan, failIfPredicateIsNotMeet(metadata, session, PERMISSION_DENIED, AccessDeniedException.PREFIX + "Cannot insert row that does not match to a row filter"), node -> {
        Scope accessControlScope = analysis.getAccessControlScope(table);
        // hidden fields are not accessible in insert
        return Scope.builder().like(accessControlScope).withRelationType(accessControlScope.getRelationId(), accessControlScope.getRelationType().withOnlyVisibleFields()).build();
    });
    List<String> insertedTableColumnNames = insertedColumns.stream().map(ColumnMetadata::getName).collect(toImmutableList());
    String catalogName = tableHandle.getCatalogName().getCatalogName();
    TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, catalogName, tableMetadata.getMetadata());
    if (materializedViewRefreshWriterTarget.isPresent()) {
        return createTableWriterPlan(analysis, plan.getRoot(), plan.getFieldMappings(), materializedViewRefreshWriterTarget.get(), insertedTableColumnNames, insertedColumns, newTableLayout, statisticsMetadata);
    }
    InsertReference insertTarget = new InsertReference(tableHandle, insertedTableColumnNames.stream().map(columns::get).collect(toImmutableList()));
    return createTableWriterPlan(analysis, plan.getRoot(), plan.getFieldMappings(), insertTarget, insertedTableColumnNames, insertedColumns, newTableLayout, statisticsMetadata);
}

Example 2 with TableStatisticsMetadata

use of io.trino.spi.statistics.TableStatisticsMetadata in project trino by trinodb.

the class LogicalPlanner method createAnalyzePlan.

private RelationPlan createAnalyzePlan(Analysis analysis, Analyze analyzeStatement) {
    TableHandle targetTable = analysis.getAnalyzeTarget().orElseThrow();
    // Plan table scan
    Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, targetTable);
    ImmutableList.Builder<Symbol> tableScanOutputs = ImmutableList.builder();
    ImmutableMap.Builder<Symbol, ColumnHandle> symbolToColumnHandle = ImmutableMap.builder();
    ImmutableMap.Builder<String, Symbol> columnNameToSymbol = ImmutableMap.builder();
    TableMetadata tableMetadata = metadata.getTableMetadata(session, targetTable);
    for (ColumnMetadata column : tableMetadata.getColumns()) {
        Symbol symbol = symbolAllocator.newSymbol(column.getName(), column.getType());
        tableScanOutputs.add(symbol);
        symbolToColumnHandle.put(symbol, columnHandles.get(column.getName()));
        columnNameToSymbol.put(column.getName(), symbol);
    }
    TableStatisticsMetadata tableStatisticsMetadata = metadata.getStatisticsCollectionMetadata(session, targetTable.getCatalogName().getCatalogName(), tableMetadata.getMetadata());
    TableStatisticAggregation tableStatisticAggregation = statisticsAggregationPlanner.createStatisticsAggregation(tableStatisticsMetadata, columnNameToSymbol.buildOrThrow());
    StatisticAggregations statisticAggregations = tableStatisticAggregation.getAggregations();
    List<Symbol> groupingSymbols = statisticAggregations.getGroupingSymbols();
    PlanNode planNode = new StatisticsWriterNode(idAllocator.getNextId(), new AggregationNode(idAllocator.getNextId(), TableScanNode.newInstance(idAllocator.getNextId(), targetTable, tableScanOutputs.build(), symbolToColumnHandle.buildOrThrow(), false, Optional.empty()), statisticAggregations.getAggregations(), singleGroupingSet(groupingSymbols), ImmutableList.of(), AggregationNode.Step.SINGLE, Optional.empty(), Optional.empty()), new StatisticsWriterNode.WriteStatisticsReference(targetTable), symbolAllocator.newSymbol("rows", BIGINT), tableStatisticsMetadata.getTableStatistics().contains(ROW_COUNT), tableStatisticAggregation.getDescriptor());
    return new RelationPlan(planNode, analysis.getScope(analyzeStatement), planNode.getOutputSymbols(), Optional.empty());
}

Example 3 with TableStatisticsMetadata

use of io.trino.spi.statistics.TableStatisticsMetadata in project trino by trinodb.

the class LogicalPlanner method createTableCreationPlan.

private RelationPlan createTableCreationPlan(Analysis analysis, Query query) {
    Analysis.Create create = analysis.getCreate().orElseThrow();
    QualifiedObjectName destination = create.getDestination().orElseThrow();
    RelationPlan plan = createRelationPlan(analysis, query);
    if (!create.isCreateTableAsSelectWithData()) {
        PlanNode root = new LimitNode(idAllocator.getNextId(), plan.getRoot(), 0L, false);
        plan = new RelationPlan(root, plan.getScope(), plan.getFieldMappings(), Optional.empty());
    }
    ConnectorTableMetadata tableMetadata = create.getMetadata().orElseThrow();
    Optional<TableLayout> newTableLayout = create.getLayout();
    List<String> columnNames = tableMetadata.getColumns().stream().filter(// todo this filter is redundant
    column -> !column.isHidden()).map(ColumnMetadata::getName).collect(toImmutableList());
    TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, destination.getCatalogName(), tableMetadata);
    return createTableWriterPlan(analysis, plan.getRoot(), visibleFields(plan), new CreateReference(destination.getCatalogName(), tableMetadata, newTableLayout), columnNames, tableMetadata.getColumns(), newTableLayout, statisticsMetadata);
}

Example 4 with TableStatisticsMetadata

use of io.trino.spi.statistics.TableStatisticsMetadata in project trino by trinodb.

the class LogicalPlanner method createTableWriterPlan.

private RelationPlan createTableWriterPlan(Analysis analysis, PlanNode source, List<Symbol> symbols, WriterTarget target, List<String> columnNames, List<ColumnMetadata> columnMetadataList, Optional<TableLayout> writeTableLayout, TableStatisticsMetadata statisticsMetadata) {
    Optional<PartitioningScheme> partitioningScheme = Optional.empty();
    Optional<PartitioningScheme> preferredPartitioningScheme = Optional.empty();
    if (writeTableLayout.isPresent()) {
        List<Symbol> partitionFunctionArguments = new ArrayList<>();
        writeTableLayout.get().getPartitionColumns().stream().mapToInt(columnNames::indexOf).mapToObj(symbols::get).forEach(partitionFunctionArguments::add);
        List<Symbol> outputLayout = new ArrayList<>(symbols);
        Optional<PartitioningHandle> partitioningHandle = writeTableLayout.get().getPartitioning();
        if (partitioningHandle.isPresent()) {
            partitioningScheme = Optional.of(new PartitioningScheme(Partitioning.create(partitioningHandle.get(), partitionFunctionArguments), outputLayout));
        } else {
            // empty connector partitioning handle means evenly partitioning on partitioning columns
            preferredPartitioningScheme = Optional.of(new PartitioningScheme(Partitioning.create(FIXED_HASH_DISTRIBUTION, partitionFunctionArguments), outputLayout));
        }
    }
    verify(columnNames.size() == symbols.size(), "columnNames.size() != symbols.size(): %s and %s", columnNames, symbols);
    Map<String, Symbol> columnToSymbolMap = zip(columnNames.stream(), symbols.stream(), SimpleImmutableEntry::new).collect(toImmutableMap(Entry::getKey, Entry::getValue));
    Set<Symbol> notNullColumnSymbols = columnMetadataList.stream().filter(column -> !column.isNullable()).map(ColumnMetadata::getName).map(columnToSymbolMap::get).collect(toImmutableSet());
    if (!statisticsMetadata.isEmpty()) {
        TableStatisticAggregation result = statisticsAggregationPlanner.createStatisticsAggregation(statisticsMetadata, columnToSymbolMap);
        StatisticAggregations.Parts aggregations = result.getAggregations().createPartialAggregations(symbolAllocator, plannerContext);
        // partial aggregation is run within the TableWriteOperator to calculate the statistics for
        // the data consumed by the TableWriteOperator
        // final aggregation is run within the TableFinishOperator to summarize collected statistics
        // by the partial aggregation from all of the writer nodes
        StatisticAggregations partialAggregation = aggregations.getPartialAggregation();
        TableFinishNode commitNode = new TableFinishNode(idAllocator.getNextId(), new TableWriterNode(idAllocator.getNextId(), source, target, symbolAllocator.newSymbol("partialrows", BIGINT), symbolAllocator.newSymbol("fragment", VARBINARY), symbols, columnNames, notNullColumnSymbols, partitioningScheme, preferredPartitioningScheme, Optional.of(partialAggregation), Optional.of(result.getDescriptor().map(aggregations.getMappings()::get))), target, symbolAllocator.newSymbol("rows", BIGINT), Optional.of(aggregations.getFinalAggregation()), Optional.of(result.getDescriptor()));
        return new RelationPlan(commitNode, analysis.getRootScope(), commitNode.getOutputSymbols(), Optional.empty());
    }
    TableFinishNode commitNode = new TableFinishNode(idAllocator.getNextId(), new TableWriterNode(idAllocator.getNextId(), source, target, symbolAllocator.newSymbol("partialrows", BIGINT), symbolAllocator.newSymbol("fragment", VARBINARY), symbols, columnNames, notNullColumnSymbols, partitioningScheme, preferredPartitioningScheme, Optional.empty(), Optional.empty()), target, symbolAllocator.newSymbol("rows", BIGINT), Optional.empty(), Optional.empty());
    return new RelationPlan(commitNode, analysis.getRootScope(), commitNode.getOutputSymbols(), Optional.empty());
}

Example 5 with TableStatisticsMetadata

use of io.trino.spi.statistics.TableStatisticsMetadata in project trino by trinodb.

the class DeltaLakeMetadata method getStatisticsCollectionMetadata.

@Override
public TableStatisticsMetadata getStatisticsCollectionMetadata(ConnectorSession session, ConnectorTableMetadata tableMetadata) {
    ImmutableSet.Builder<ColumnStatisticMetadata> columnStatistics = ImmutableSet.builder();
    Optional<Set<String>> analyzeColumnNames = DeltaLakeTableProperties.getAnalyzeColumns(tableMetadata.getProperties());
    tableMetadata.getColumns().stream().filter(DeltaLakeMetadata::shouldCollectExtendedStatistics).filter(columnMetadata -> analyzeColumnNames.map(columnNames -> columnNames.contains(columnMetadata.getName())).orElse(true)).map(columnMetadata -> new ColumnStatisticMetadata(columnMetadata.getName(), NUMBER_OF_DISTINCT_VALUES_SUMMARY)).forEach(columnStatistics::add);
    // collect max(file modification time) for sake of incremental ANALYZE
    columnStatistics.add(new ColumnStatisticMetadata(FILE_MODIFIED_TIME_COLUMN_NAME, MAX_VALUE));
    return new TableStatisticsMetadata(columnStatistics.build(), ImmutableSet.of(), ImmutableList.of());
}