Examples with TableStatisticsMetadata com.facebook.presto.spi.statistics.TableStatisticsMetadata used on opensource projects

Example 1 with TableStatisticsMetadata

use of com.facebook.presto.spi.statistics.TableStatisticsMetadata in project presto by prestodb.

the class LogicalPlanner method createTableCreationPlan.

private RelationPlan createTableCreationPlan(Analysis analysis, Query query) {
    QualifiedObjectName destination = analysis.getCreateTableDestination().get();
    RelationPlan plan = createRelationPlan(analysis, query);
    ConnectorTableMetadata tableMetadata = createTableMetadata(destination, getOutputTableColumns(plan, analysis.getColumnAliases()), analysis.getCreateTableProperties(), analysis.getParameters(), analysis.getCreateTableComment());
    Optional<NewTableLayout> newTableLayout = metadata.getNewTableLayout(session, destination.getCatalogName(), tableMetadata);
    Optional<NewTableLayout> preferredShuffleLayout = metadata.getPreferredShuffleLayoutForNewTable(session, destination.getCatalogName(), tableMetadata);
    List<String> columnNames = tableMetadata.getColumns().stream().filter(column -> !column.isHidden()).map(ColumnMetadata::getName).collect(toImmutableList());
    TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, destination.getCatalogName(), tableMetadata);
    return createTableWriterPlan(analysis, plan, new CreateName(new ConnectorId(destination.getCatalogName()), tableMetadata, newTableLayout), columnNames, tableMetadata.getColumns(), newTableLayout, preferredShuffleLayout, statisticsMetadata);
}

Example 2 with TableStatisticsMetadata

use of com.facebook.presto.spi.statistics.TableStatisticsMetadata in project presto by prestodb.

the class LogicalPlanner method createTableWriterPlan.

private RelationPlan createTableWriterPlan(Analysis analysis, RelationPlan plan, WriterTarget target, List<String> columnNames, List<ColumnMetadata> columnMetadataList, Optional<NewTableLayout> writeTableLayout, Optional<NewTableLayout> preferredShuffleLayout, TableStatisticsMetadata statisticsMetadata) {
    verify(!(writeTableLayout.isPresent() && preferredShuffleLayout.isPresent()), "writeTableLayout and preferredShuffleLayout cannot both exist");
    PlanNode source = plan.getRoot();
    if (!analysis.isCreateTableAsSelectWithData()) {
        source = new LimitNode(source.getSourceLocation(), idAllocator.getNextId(), source, 0L, FINAL);
    }
    List<VariableReferenceExpression> variables = plan.getFieldMappings();
    Optional<PartitioningScheme> tablePartitioningScheme = getPartitioningSchemeForTableWrite(writeTableLayout, columnNames, variables);
    Optional<PartitioningScheme> preferredShufflePartitioningScheme = getPartitioningSchemeForTableWrite(preferredShuffleLayout, columnNames, variables);
    verify(columnNames.size() == variables.size(), "columnNames.size() != variables.size(): %s and %s", columnNames, variables);
    Map<String, VariableReferenceExpression> columnToVariableMap = zip(columnNames.stream(), plan.getFieldMappings().stream(), SimpleImmutableEntry::new).collect(toImmutableMap(Entry::getKey, Entry::getValue));
    Set<VariableReferenceExpression> notNullColumnVariables = columnMetadataList.stream().filter(column -> !column.isNullable()).map(ColumnMetadata::getName).map(columnToVariableMap::get).collect(toImmutableSet());
    if (!statisticsMetadata.isEmpty()) {
        TableStatisticAggregation result = statisticsAggregationPlanner.createStatisticsAggregation(statisticsMetadata, columnToVariableMap, true);
        StatisticAggregations.Parts aggregations = result.getAggregations().splitIntoPartialAndFinal(variableAllocator, metadata.getFunctionAndTypeManager());
        TableFinishNode commitNode = new TableFinishNode(source.getSourceLocation(), idAllocator.getNextId(), new TableWriterNode(source.getSourceLocation(), idAllocator.getNextId(), source, Optional.of(target), variableAllocator.newVariable("rows", BIGINT), variableAllocator.newVariable("fragments", VARBINARY), variableAllocator.newVariable("commitcontext", VARBINARY), plan.getFieldMappings(), columnNames, notNullColumnVariables, tablePartitioningScheme, preferredShufflePartitioningScheme, // the data consumed by the TableWriteOperator
        Optional.of(aggregations.getPartialAggregation())), Optional.of(target), variableAllocator.newVariable("rows", BIGINT), // by the partial aggregation from all of the writer nodes
        Optional.of(aggregations.getFinalAggregation()), Optional.of(result.getDescriptor()));
        return new RelationPlan(commitNode, analysis.getRootScope(), commitNode.getOutputVariables());
    }
    TableFinishNode commitNode = new TableFinishNode(source.getSourceLocation(), idAllocator.getNextId(), new TableWriterNode(source.getSourceLocation(), idAllocator.getNextId(), source, Optional.of(target), variableAllocator.newVariable("rows", BIGINT), variableAllocator.newVariable("fragments", VARBINARY), variableAllocator.newVariable("commitcontext", VARBINARY), plan.getFieldMappings(), columnNames, notNullColumnVariables, tablePartitioningScheme, preferredShufflePartitioningScheme, Optional.empty()), Optional.of(target), variableAllocator.newVariable("rows", BIGINT), Optional.empty(), Optional.empty());
    return new RelationPlan(commitNode, analysis.getRootScope(), commitNode.getOutputVariables());
}

Example 3 with TableStatisticsMetadata

use of com.facebook.presto.spi.statistics.TableStatisticsMetadata in project presto by prestodb.

the class LogicalPlanner method buildInternalInsertPlan.

private RelationPlan buildInternalInsertPlan(TableHandle tableHandle, List<ColumnHandle> columnHandles, Query query, Analysis analysis, WriterTarget target) {
    TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle);
    List<ColumnMetadata> visibleTableColumns = tableMetadata.getColumns().stream().filter(column -> !column.isHidden()).collect(toImmutableList());
    List<String> visibleTableColumnNames = visibleTableColumns.stream().map(ColumnMetadata::getName).collect(toImmutableList());
    RelationPlan plan = createRelationPlan(analysis, query);
    Map<String, ColumnHandle> columns = metadata.getColumnHandles(session, tableHandle);
    Assignments.Builder assignments = Assignments.builder();
    for (ColumnMetadata column : tableMetadata.getColumns()) {
        if (column.isHidden()) {
            continue;
        }
        VariableReferenceExpression output = variableAllocator.newVariable(getSourceLocation(query), column.getName(), column.getType());
        int index = columnHandles.indexOf(columns.get(column.getName()));
        if (index < 0) {
            Expression cast = new Cast(new NullLiteral(), column.getType().getTypeSignature().toString());
            assignments.put(output, castToRowExpression(cast));
        } else {
            VariableReferenceExpression input = plan.getVariable(index);
            Type tableType = column.getType();
            Type queryType = input.getType();
            if (queryType.equals(tableType) || metadata.getFunctionAndTypeManager().isTypeOnlyCoercion(queryType, tableType)) {
                assignments.put(output, castToRowExpression(createSymbolReference(input)));
            } else {
                Expression cast = new Cast(createSymbolReference(input), tableType.getTypeSignature().toString());
                assignments.put(output, castToRowExpression(cast));
            }
        }
    }
    ProjectNode projectNode = new ProjectNode(idAllocator.getNextId(), plan.getRoot(), assignments.build());
    List<Field> fields = visibleTableColumns.stream().map(column -> Field.newUnqualified(query.getLocation(), column.getName(), column.getType())).collect(toImmutableList());
    Scope scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(fields)).build();
    plan = new RelationPlan(projectNode, scope, projectNode.getOutputVariables());
    Optional<NewTableLayout> newTableLayout = metadata.getInsertLayout(session, tableHandle);
    Optional<NewTableLayout> preferredShuffleLayout = metadata.getPreferredShuffleLayoutForInsert(session, tableHandle);
    String catalogName = tableHandle.getConnectorId().getCatalogName();
    TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, catalogName, tableMetadata.getMetadata());
    return createTableWriterPlan(analysis, plan, target, visibleTableColumnNames, visibleTableColumns, newTableLayout, preferredShuffleLayout, statisticsMetadata);
}

Example 4 with TableStatisticsMetadata

use of com.facebook.presto.spi.statistics.TableStatisticsMetadata in project presto by prestodb.

the class LogicalPlanner method createAnalyzePlan.

private RelationPlan createAnalyzePlan(Analysis analysis, Analyze analyzeStatement) {
    TableHandle targetTable = analysis.getAnalyzeTarget().get();
    // Plan table scan
    Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, targetTable);
    ImmutableList.Builder<VariableReferenceExpression> tableScanOutputsBuilder = ImmutableList.builder();
    ImmutableMap.Builder<VariableReferenceExpression, ColumnHandle> variableToColumnHandle = ImmutableMap.builder();
    ImmutableMap.Builder<String, VariableReferenceExpression> columnNameToVariable = ImmutableMap.builder();
    TableMetadata tableMetadata = metadata.getTableMetadata(session, targetTable);
    for (ColumnMetadata column : tableMetadata.getColumns()) {
        VariableReferenceExpression variable = variableAllocator.newVariable(getSourceLocation(analyzeStatement), column.getName(), column.getType());
        tableScanOutputsBuilder.add(variable);
        variableToColumnHandle.put(variable, columnHandles.get(column.getName()));
        columnNameToVariable.put(column.getName(), variable);
    }
    List<VariableReferenceExpression> tableScanOutputs = tableScanOutputsBuilder.build();
    TableStatisticsMetadata tableStatisticsMetadata = metadata.getStatisticsCollectionMetadata(session, targetTable.getConnectorId().getCatalogName(), tableMetadata.getMetadata());
    TableStatisticAggregation tableStatisticAggregation = statisticsAggregationPlanner.createStatisticsAggregation(tableStatisticsMetadata, columnNameToVariable.build(), true);
    StatisticAggregations statisticAggregations = tableStatisticAggregation.getAggregations();
    PlanNode planNode = new StatisticsWriterNode(getSourceLocation(analyzeStatement), idAllocator.getNextId(), new AggregationNode(getSourceLocation(analyzeStatement), idAllocator.getNextId(), new TableScanNode(getSourceLocation(analyzeStatement), idAllocator.getNextId(), targetTable, tableScanOutputs, variableToColumnHandle.build(), TupleDomain.all(), TupleDomain.all()), statisticAggregations.getAggregations(), singleGroupingSet(statisticAggregations.getGroupingVariables()), ImmutableList.of(), AggregationNode.Step.SINGLE, Optional.empty(), Optional.empty()), targetTable, variableAllocator.newVariable(getSourceLocation(analyzeStatement), "rows", BIGINT), tableStatisticsMetadata.getTableStatistics().contains(ROW_COUNT), tableStatisticAggregation.getDescriptor());
    return new RelationPlan(planNode, analysis.getScope(analyzeStatement), planNode.getOutputVariables());
}