Examples with TableStatisticAggregation io.prestosql.sql.planner.StatisticsAggregationPlanner.TableStatisticAggregation used on opensource projects

Example 1 with TableStatisticAggregation

use of io.prestosql.sql.planner.StatisticsAggregationPlanner.TableStatisticAggregation in project hetu-core by openlookeng.

the class LogicalPlanner method createTableWriterPlan.

private RelationPlan createTableWriterPlan(Analysis analysis, RelationPlan plan, WriterTarget target, List<String> columnNames, Optional<NewTableLayout> writeTableLayout, TableStatisticsMetadata statisticsMetadata) {
    PlanNode source = plan.getRoot();
    if (!analysis.isCreateTableAsSelectWithData()) {
        source = new LimitNode(idAllocator.getNextId(), source, 0L, false);
    }
    // todo this should be checked in analysis
    writeTableLayout.ifPresent(layout -> {
        if (!ImmutableSet.copyOf(columnNames).containsAll(layout.getPartitionColumns())) {
            throw new PrestoException(NOT_SUPPORTED, "INSERT must write all distribution columns: " + layout.getPartitionColumns());
        }
    });
    List<Symbol> symbols = plan.getFieldMappings();
    Optional<PartitioningScheme> partitioningScheme = Optional.empty();
    if (writeTableLayout.isPresent()) {
        List<Symbol> partitionFunctionArguments = new ArrayList<>();
        writeTableLayout.get().getPartitionColumns().stream().mapToInt(columnNames::indexOf).mapToObj(symbols::get).forEach(partitionFunctionArguments::add);
        List<Symbol> outputLayout = new ArrayList<>(symbols);
        PartitioningHandle partitioningHandle = writeTableLayout.get().getPartitioning().orElse(FIXED_HASH_DISTRIBUTION);
        partitioningScheme = Optional.of(new PartitioningScheme(Partitioning.create(partitioningHandle, partitionFunctionArguments), outputLayout));
    }
    if (!statisticsMetadata.isEmpty()) {
        verify(columnNames.size() == symbols.size(), "columnNames.size() != symbols.size(): %s and %s", columnNames, symbols);
        Map<String, Symbol> columnToSymbolMap = zip(columnNames.stream(), symbols.stream(), SimpleImmutableEntry::new).collect(toImmutableMap(Entry::getKey, Entry::getValue));
        TableStatisticAggregation result = statisticsAggregationPlanner.createStatisticsAggregation(statisticsMetadata, columnToSymbolMap);
        StatisticAggregations.Parts aggregations = result.getAggregations().createPartialAggregations(planSymbolAllocator, metadata);
        // partial aggregation is run within the TableWriteOperator to calculate the statistics for
        // the data consumed by the TableWriteOperator
        // final aggregation is run within the TableFinishOperator to summarize collected statistics
        // by the partial aggregation from all of the writer nodes
        StatisticAggregations partialAggregation = aggregations.getPartialAggregation();
        PlanNode writerNode = new TableWriterNode(idAllocator.getNextId(), source, target, planSymbolAllocator.newSymbol("partialrows", BIGINT), planSymbolAllocator.newSymbol("fragment", VARBINARY), symbols, columnNames, partitioningScheme, Optional.of(partialAggregation), Optional.of(result.getDescriptor().map(aggregations.getMappings()::get)));
        TableFinishNode commitNode = new TableFinishNode(idAllocator.getNextId(), writerNode, target, planSymbolAllocator.newSymbol("rows", BIGINT), Optional.of(aggregations.getFinalAggregation()), Optional.of(result.getDescriptor()));
        return new RelationPlan(commitNode, analysis.getRootScope(), commitNode.getOutputSymbols());
    }
    TableFinishNode commitNode = new TableFinishNode(idAllocator.getNextId(), new TableWriterNode(idAllocator.getNextId(), source, target, planSymbolAllocator.newSymbol("partialrows", BIGINT), planSymbolAllocator.newSymbol("fragment", VARBINARY), symbols, columnNames, partitioningScheme, Optional.empty(), Optional.empty()), target, planSymbolAllocator.newSymbol("rows", BIGINT), Optional.empty(), Optional.empty());
    return new RelationPlan(commitNode, analysis.getRootScope(), commitNode.getOutputSymbols());
}

Example 2 with TableStatisticAggregation

use of io.prestosql.sql.planner.StatisticsAggregationPlanner.TableStatisticAggregation in project hetu-core by openlookeng.

the class LogicalPlanner method createVacuumWriterPlan.

private RelationPlan createVacuumWriterPlan(Analysis analysis, TableHandle handle, VacuumTable node, WriterTarget target, List<Symbol> symbols, List<String> columnNames, TableStatisticsMetadata statisticsMetadata) {
    Optional<StatisticAggregations> statisticsAggregation = Optional.empty();
    Optional<StatisticAggregationsDescriptor<Symbol>> statisticsAggregationDescriptor = Optional.empty();
    Optional<StatisticAggregations> finalStatisticsAggregation = Optional.empty();
    Optional<StatisticAggregationsDescriptor<Symbol>> finalStatisticsAggregationDescriptor = Optional.empty();
    if (!statisticsMetadata.isEmpty()) {
        verify(columnNames.size() == symbols.size(), "columnNames.size() != symbols.size(): %s and %s", columnNames, symbols);
        Map<String, Symbol> columnToSymbolMap = zip(columnNames.stream(), symbols.stream(), SimpleImmutableEntry::new).collect(toImmutableMap(Entry::getKey, Entry::getValue));
        TableStatisticAggregation result = statisticsAggregationPlanner.createStatisticsAggregation(statisticsMetadata, columnToSymbolMap);
        StatisticAggregations.Parts aggregations = result.getAggregations().createPartialAggregations(planSymbolAllocator, metadata);
        // partial aggregation is run within the VacuumTableOperator to calculate the statistics for
        // the data consumed by the VacuumTableOperator
        // final aggregation is run within the TableFinishOperator to summarize collected statistics
        // by the partial aggregation from all of the writer nodes
        statisticsAggregation = Optional.of(aggregations.getPartialAggregation());
        statisticsAggregationDescriptor = Optional.of(result.getDescriptor().map(aggregations.getMappings()::get));
        finalStatisticsAggregation = Optional.of(aggregations.getFinalAggregation());
        finalStatisticsAggregationDescriptor = Optional.of(result.getDescriptor());
    }
    VacuumTableNode vacuumTableNode = new VacuumTableNode(idAllocator.getNextId(), handle, target, planSymbolAllocator.newSymbol("partialrows", BIGINT), planSymbolAllocator.newSymbol("fragment", VARBINARY), node.getPartition().orElse(""), node.isFull(), symbols, statisticsAggregation, statisticsAggregationDescriptor);
    TableFinishNode commitNode = new TableFinishNode(idAllocator.getNextId(), vacuumTableNode, target, planSymbolAllocator.newSymbol("rows", BIGINT), finalStatisticsAggregation, finalStatisticsAggregationDescriptor);
    return new RelationPlan(commitNode, analysis.getRootScope(), commitNode.getOutputSymbols());
}

Example 3 with TableStatisticAggregation

use of io.prestosql.sql.planner.StatisticsAggregationPlanner.TableStatisticAggregation in project hetu-core by openlookeng.

the class LogicalPlanner method createAnalyzePlan.

private RelationPlan createAnalyzePlan(Analysis analysis, Analyze analyzeStatement) {
    TableHandle targetTable = analysis.getAnalyzeTarget().get();
    // Plan table scan
    Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, targetTable);
    ImmutableList.Builder<Symbol> tableScanOutputs = ImmutableList.builder();
    ImmutableMap.Builder<Symbol, ColumnHandle> symbolToColumnHandle = ImmutableMap.builder();
    ImmutableMap.Builder<String, Symbol> columnNameToSymbol = ImmutableMap.builder();
    TableMetadata tableMetadata = metadata.getTableMetadata(session, targetTable);
    for (ColumnMetadata column : tableMetadata.getColumns()) {
        Symbol symbol = planSymbolAllocator.newSymbol(column.getName(), column.getType());
        tableScanOutputs.add(symbol);
        symbolToColumnHandle.put(symbol, columnHandles.get(column.getName()));
        columnNameToSymbol.put(column.getName(), symbol);
    }
    TableStatisticsMetadata tableStatisticsMetadata = metadata.getStatisticsCollectionMetadata(session, targetTable.getCatalogName().getCatalogName(), tableMetadata.getMetadata());
    TableStatisticAggregation tableStatisticAggregation = statisticsAggregationPlanner.createStatisticsAggregation(tableStatisticsMetadata, columnNameToSymbol.build());
    StatisticAggregations statisticAggregations = tableStatisticAggregation.getAggregations();
    List<Symbol> groupingSymbols = statisticAggregations.getGroupingSymbols();
    PlanNode planNode = new StatisticsWriterNode(idAllocator.getNextId(), new AggregationNode(idAllocator.getNextId(), TableScanNode.newInstance(idAllocator.getNextId(), targetTable, tableScanOutputs.build(), symbolToColumnHandle.build(), ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), 0, false), statisticAggregations.getAggregations(), singleGroupingSet(groupingSymbols), ImmutableList.of(), AggregationNode.Step.SINGLE, Optional.empty(), Optional.empty(), AggregationNode.AggregationType.HASH, Optional.empty()), new StatisticsWriterNode.WriteStatisticsReference(targetTable), planSymbolAllocator.newSymbol("rows", BIGINT), tableStatisticsMetadata.getTableStatistics().contains(ROW_COUNT), tableStatisticAggregation.getDescriptor());
    return new RelationPlan(planNode, analysis.getScope(analyzeStatement), planNode.getOutputSymbols());
}