Examples with TableWriterNode com.facebook.presto.sql.planner.plan.TableWriterNode used on opensource projects

Example 1 with TableWriterNode

use of com.facebook.presto.sql.planner.plan.TableWriterNode in project presto by prestodb.

the class TestVerifyNoOriginalExpression method testTableWriter.

@Test
public void testTableWriter() {
    ImmutableList<VariableReferenceExpression> variableReferenceExpressions = ImmutableList.of(VARIABLE_REFERENCE_EXPRESSION);
    TableWriterNode tableWriterNode = builder.tableWriter(variableReferenceExpressions, ImmutableList.of(""), valuesNode);
    testValidation(tableWriterNode);
}

Example 2 with TableWriterNode

use of com.facebook.presto.sql.planner.plan.TableWriterNode in project presto by prestodb.

the class LogicalPlanner method createTableWriterPlan.

private RelationPlan createTableWriterPlan(Analysis analysis, RelationPlan plan, WriterTarget target, List<String> columnNames, List<ColumnMetadata> columnMetadataList, Optional<NewTableLayout> writeTableLayout, Optional<NewTableLayout> preferredShuffleLayout, TableStatisticsMetadata statisticsMetadata) {
    verify(!(writeTableLayout.isPresent() && preferredShuffleLayout.isPresent()), "writeTableLayout and preferredShuffleLayout cannot both exist");
    PlanNode source = plan.getRoot();
    if (!analysis.isCreateTableAsSelectWithData()) {
        source = new LimitNode(source.getSourceLocation(), idAllocator.getNextId(), source, 0L, FINAL);
    }
    List<VariableReferenceExpression> variables = plan.getFieldMappings();
    Optional<PartitioningScheme> tablePartitioningScheme = getPartitioningSchemeForTableWrite(writeTableLayout, columnNames, variables);
    Optional<PartitioningScheme> preferredShufflePartitioningScheme = getPartitioningSchemeForTableWrite(preferredShuffleLayout, columnNames, variables);
    verify(columnNames.size() == variables.size(), "columnNames.size() != variables.size(): %s and %s", columnNames, variables);
    Map<String, VariableReferenceExpression> columnToVariableMap = zip(columnNames.stream(), plan.getFieldMappings().stream(), SimpleImmutableEntry::new).collect(toImmutableMap(Entry::getKey, Entry::getValue));
    Set<VariableReferenceExpression> notNullColumnVariables = columnMetadataList.stream().filter(column -> !column.isNullable()).map(ColumnMetadata::getName).map(columnToVariableMap::get).collect(toImmutableSet());
    if (!statisticsMetadata.isEmpty()) {
        TableStatisticAggregation result = statisticsAggregationPlanner.createStatisticsAggregation(statisticsMetadata, columnToVariableMap, true);
        StatisticAggregations.Parts aggregations = result.getAggregations().splitIntoPartialAndFinal(variableAllocator, metadata.getFunctionAndTypeManager());
        TableFinishNode commitNode = new TableFinishNode(source.getSourceLocation(), idAllocator.getNextId(), new TableWriterNode(source.getSourceLocation(), idAllocator.getNextId(), source, Optional.of(target), variableAllocator.newVariable("rows", BIGINT), variableAllocator.newVariable("fragments", VARBINARY), variableAllocator.newVariable("commitcontext", VARBINARY), plan.getFieldMappings(), columnNames, notNullColumnVariables, tablePartitioningScheme, preferredShufflePartitioningScheme, // the data consumed by the TableWriteOperator
        Optional.of(aggregations.getPartialAggregation())), Optional.of(target), variableAllocator.newVariable("rows", BIGINT), // by the partial aggregation from all of the writer nodes
        Optional.of(aggregations.getFinalAggregation()), Optional.of(result.getDescriptor()));
        return new RelationPlan(commitNode, analysis.getRootScope(), commitNode.getOutputVariables());
    }
    TableFinishNode commitNode = new TableFinishNode(source.getSourceLocation(), idAllocator.getNextId(), new TableWriterNode(source.getSourceLocation(), idAllocator.getNextId(), source, Optional.of(target), variableAllocator.newVariable("rows", BIGINT), variableAllocator.newVariable("fragments", VARBINARY), variableAllocator.newVariable("commitcontext", VARBINARY), plan.getFieldMappings(), columnNames, notNullColumnVariables, tablePartitioningScheme, preferredShufflePartitioningScheme, Optional.empty()), Optional.of(target), variableAllocator.newVariable("rows", BIGINT), Optional.empty(), Optional.empty());
    return new RelationPlan(commitNode, analysis.getRootScope(), commitNode.getOutputVariables());
}

Example 3 with TableWriterNode

use of com.facebook.presto.sql.planner.plan.TableWriterNode in project presto by prestodb.

the class TableWriterMatcher method detailMatches.

@Override
public MatchResult detailMatches(PlanNode node, StatsProvider stats, Session session, Metadata metadata, SymbolAliases symbolAliases) {
    checkState(shapeMatches(node), "Plan testing framework error: shapeMatches returned false in detailMatches in %s", this.getClass().getName());
    TableWriterNode tableWriterNode = (TableWriterNode) node;
    if (!tableWriterNode.getColumnNames().equals(columnNames)) {
        return NO_MATCH;
    }
    if (!columns.stream().map(s -> Symbol.from(symbolAliases.get(s))).collect(toImmutableList()).equals(tableWriterNode.getColumns().stream().map(VariableReferenceExpression::getName).map(Symbol::new).collect(toImmutableList()))) {
        return NO_MATCH;
    }
    return match();
}

Example 4 with TableWriterNode

use of com.facebook.presto.sql.planner.plan.TableWriterNode in project presto by prestodb.

the class LogicalPlanner method createTableWriterPlan.

private RelationPlan createTableWriterPlan(Analysis analysis, RelationPlan plan, WriterTarget target, List<String> columnNames, Optional<NewTableLayout> writeTableLayout) {
    List<Symbol> writerOutputs = ImmutableList.of(symbolAllocator.newSymbol("partialrows", BIGINT), symbolAllocator.newSymbol("fragment", VARBINARY));
    PlanNode source = plan.getRoot();
    if (!analysis.isCreateTableAsSelectWithData()) {
        source = new LimitNode(idAllocator.getNextId(), source, 0L, false);
    }
    // todo this should be checked in analysis
    writeTableLayout.ifPresent(layout -> {
        if (!ImmutableSet.copyOf(columnNames).containsAll(layout.getPartitionColumns())) {
            throw new PrestoException(NOT_SUPPORTED, "INSERT must write all distribution columns: " + layout.getPartitionColumns());
        }
    });
    List<Symbol> symbols = plan.getFieldMappings();
    Optional<PartitioningScheme> partitioningScheme = Optional.empty();
    if (writeTableLayout.isPresent()) {
        List<Symbol> partitionFunctionArguments = new ArrayList<>();
        writeTableLayout.get().getPartitionColumns().stream().mapToInt(columnNames::indexOf).mapToObj(symbols::get).forEach(partitionFunctionArguments::add);
        List<Symbol> outputLayout = new ArrayList<>(symbols);
        partitioningScheme = Optional.of(new PartitioningScheme(Partitioning.create(writeTableLayout.get().getPartitioning(), partitionFunctionArguments), outputLayout));
    }
    PlanNode writerNode = new TableWriterNode(idAllocator.getNextId(), source, target, symbols, columnNames, writerOutputs, partitioningScheme);
    List<Symbol> outputs = ImmutableList.of(symbolAllocator.newSymbol("rows", BIGINT));
    TableFinishNode commitNode = new TableFinishNode(idAllocator.getNextId(), writerNode, target, outputs);
    return new RelationPlan(commitNode, analysis.getRootScope(), outputs);
}

Example 5 with TableWriterNode

use of com.facebook.presto.sql.planner.plan.TableWriterNode in project presto by prestodb.

the class PlanFragmenter method analyzeGroupedExecution.

/*
     * In theory, recoverable grouped execution should be decided at query section level (i.e. a connected component of stages connected by remote exchanges).
     * This is because supporting mixed recoverable execution and non-recoverable execution within a query section adds unnecessary complications but provides little benefit,
     * because a single task failure is still likely to fail the non-recoverable stage.
     * However, since the concept of "query section" is not introduced until execution time as of now, it needs significant hacks to decide at fragmenting time.

     * TODO: We should introduce "query section" and make recoverability analysis done at query section level.
     */
private SubPlan analyzeGroupedExecution(Session session, SubPlan subPlan, boolean parentContainsTableFinish) {
    PlanFragment fragment = subPlan.getFragment();
    GroupedExecutionProperties properties = fragment.getRoot().accept(new GroupedExecutionTagger(session, metadata, nodePartitioningManager), null);
    if (properties.isSubTreeUseful()) {
        boolean preferDynamic = fragment.getRemoteSourceNodes().stream().allMatch(node -> node.getExchangeType() == REPLICATE);
        BucketNodeMap bucketNodeMap = nodePartitioningManager.getBucketNodeMap(session, fragment.getPartitioning(), preferDynamic);
        if (bucketNodeMap.isDynamic()) {
            /*
                 * We currently only support recoverable grouped execution if the following statements hold true:
                 *   - Current session enables recoverable grouped execution and table writer merge operator
                 *   - Parent sub plan contains TableFinishNode
                 *   - Current sub plan's root is TableWriterMergeNode or TableWriterNode
                 *   - Input connectors supports split source rewind
                 *   - Output connectors supports partition commit
                 *   - Bucket node map uses dynamic scheduling
                 *   - One table writer per task
                 */
            boolean recoverable = isRecoverableGroupedExecutionEnabled(session) && isTableWriterMergeOperatorEnabled(session) && parentContainsTableFinish && (fragment.getRoot() instanceof TableWriterMergeNode || fragment.getRoot() instanceof TableWriterNode) && properties.isRecoveryEligible();
            if (recoverable) {
                fragment = fragment.withRecoverableGroupedExecution(properties.getCapableTableScanNodes(), properties.getTotalLifespans());
            } else {
                fragment = fragment.withDynamicLifespanScheduleGroupedExecution(properties.getCapableTableScanNodes(), properties.getTotalLifespans());
            }
        } else {
            fragment = fragment.withFixedLifespanScheduleGroupedExecution(properties.getCapableTableScanNodes(), properties.getTotalLifespans());
        }
    }
    ImmutableList.Builder<SubPlan> result = ImmutableList.builder();
    boolean containsTableFinishNode = containsTableFinishNode(fragment);
    for (SubPlan child : subPlan.getChildren()) {
        result.add(analyzeGroupedExecution(session, child, containsTableFinishNode));
    }
    return new SubPlan(fragment, result.build());
}