use of io.prestosql.sql.planner.plan.TableFinishNode in project hetu-core by openlookeng.
the class LogicalPlanner method createTableWriterPlan.
private RelationPlan createTableWriterPlan(Analysis analysis, RelationPlan plan, WriterTarget target, List<String> columnNames, Optional<NewTableLayout> writeTableLayout, TableStatisticsMetadata statisticsMetadata) {
PlanNode source = plan.getRoot();
if (!analysis.isCreateTableAsSelectWithData()) {
source = new LimitNode(idAllocator.getNextId(), source, 0L, false);
}
// todo this should be checked in analysis
writeTableLayout.ifPresent(layout -> {
if (!ImmutableSet.copyOf(columnNames).containsAll(layout.getPartitionColumns())) {
throw new PrestoException(NOT_SUPPORTED, "INSERT must write all distribution columns: " + layout.getPartitionColumns());
}
});
List<Symbol> symbols = plan.getFieldMappings();
Optional<PartitioningScheme> partitioningScheme = Optional.empty();
if (writeTableLayout.isPresent()) {
List<Symbol> partitionFunctionArguments = new ArrayList<>();
writeTableLayout.get().getPartitionColumns().stream().mapToInt(columnNames::indexOf).mapToObj(symbols::get).forEach(partitionFunctionArguments::add);
List<Symbol> outputLayout = new ArrayList<>(symbols);
PartitioningHandle partitioningHandle = writeTableLayout.get().getPartitioning().orElse(FIXED_HASH_DISTRIBUTION);
partitioningScheme = Optional.of(new PartitioningScheme(Partitioning.create(partitioningHandle, partitionFunctionArguments), outputLayout));
}
if (!statisticsMetadata.isEmpty()) {
verify(columnNames.size() == symbols.size(), "columnNames.size() != symbols.size(): %s and %s", columnNames, symbols);
Map<String, Symbol> columnToSymbolMap = zip(columnNames.stream(), symbols.stream(), SimpleImmutableEntry::new).collect(toImmutableMap(Entry::getKey, Entry::getValue));
TableStatisticAggregation result = statisticsAggregationPlanner.createStatisticsAggregation(statisticsMetadata, columnToSymbolMap);
StatisticAggregations.Parts aggregations = result.getAggregations().createPartialAggregations(planSymbolAllocator, metadata);
// partial aggregation is run within the TableWriteOperator to calculate the statistics for
// the data consumed by the TableWriteOperator
// final aggregation is run within the TableFinishOperator to summarize collected statistics
// by the partial aggregation from all of the writer nodes
StatisticAggregations partialAggregation = aggregations.getPartialAggregation();
PlanNode writerNode = new TableWriterNode(idAllocator.getNextId(), source, target, planSymbolAllocator.newSymbol("partialrows", BIGINT), planSymbolAllocator.newSymbol("fragment", VARBINARY), symbols, columnNames, partitioningScheme, Optional.of(partialAggregation), Optional.of(result.getDescriptor().map(aggregations.getMappings()::get)));
TableFinishNode commitNode = new TableFinishNode(idAllocator.getNextId(), writerNode, target, planSymbolAllocator.newSymbol("rows", BIGINT), Optional.of(aggregations.getFinalAggregation()), Optional.of(result.getDescriptor()));
return new RelationPlan(commitNode, analysis.getRootScope(), commitNode.getOutputSymbols());
}
TableFinishNode commitNode = new TableFinishNode(idAllocator.getNextId(), new TableWriterNode(idAllocator.getNextId(), source, target, planSymbolAllocator.newSymbol("partialrows", BIGINT), planSymbolAllocator.newSymbol("fragment", VARBINARY), symbols, columnNames, partitioningScheme, Optional.empty(), Optional.empty()), target, planSymbolAllocator.newSymbol("rows", BIGINT), Optional.empty(), Optional.empty());
return new RelationPlan(commitNode, analysis.getRootScope(), commitNode.getOutputSymbols());
}
use of io.prestosql.sql.planner.plan.TableFinishNode in project hetu-core by openlookeng.
the class PushDeleteAsInsertIntoConnector method apply.
@Override
public Result apply(TableFinishNode node, Captures captures, Context context) {
if (!withFilter) {
TableScanNode tableScan = captures.get(TABLE_SCAN);
return metadata.applyDelete(context.getSession(), tableScan.getTable()).map(newHandle -> new TableDeleteNode(context.getIdAllocator().getNextId(), newHandle, getOnlyElement(node.getOutputSymbols()))).map(Result::ofPlanNode).orElseGet(Result::empty);
}
TableWriterNode writerNode = captures.get(WRITER_NODE);
TableWriterNode.DeleteAsInsertReference deleteTargetRef = (TableWriterNode.DeleteAsInsertReference) writerNode.getTarget();
if (!deleteTargetRef.getConstraint().isPresent()) {
// Not expected to reach here.
return Result.empty();
}
Expression predicate = deleteTargetRef.getConstraint().get();
Expression filtered = ExpressionUtils.filterDeterministicConjuncts(predicate);
if (!predicate.equals(filtered)) {
// There were some non-deterministic filters.. so cannot directly delete
return Result.empty();
}
Set<Symbol> allPredicateSymbols = SymbolsExtractor.extractUnique(predicate);
Map<Symbol, ColumnHandle> columnAssignments = deleteTargetRef.getColumnAssignments();
Set<Symbol> allColumns = columnAssignments.keySet();
List<Symbol> predicateColumnSymbols = allPredicateSymbols.stream().filter(allColumns::contains).distinct().collect(Collectors.toList());
// If all predicate symbols are partitionColumns, then only partition can be deleted directly.
if (predicateColumnSymbols.isEmpty() || !predicateColumnSymbols.stream().allMatch(symbol -> {
ColumnHandle columnHandle = columnAssignments.get(symbol);
return columnHandle != null && columnHandle.isPartitionKey();
})) {
return Result.empty();
}
FilterNode filterNode = captures.get(FILTER);
List<Symbol> nonTableSymbols = allPredicateSymbols.stream().filter(symbol -> !allColumns.contains(symbol)).collect(Collectors.toList());
PredicateContext predicateContext = new PredicateContext();
PlanNode rewrittenSource = SimplePlanRewriter.rewriteWith(new ReWriter(columnAssignments.keySet(), nonTableSymbols, context.getLookup(), logicalRowExpressions), filterNode, predicateContext);
/**
* Create the TableDeleteNode with source to evaluate the predicate subqueries
*/
TableDeleteNode tableDeleteNode = new TableDeleteNode(context.getIdAllocator().getNextId(), rewrittenSource, Optional.of(predicateContext.tablePredicate), deleteTargetRef.getHandle(), deleteTargetRef.getColumnAssignments(), getOnlyElement(node.getOutputSymbols()));
return Result.ofPlanNode(tableDeleteNode);
}
use of io.prestosql.sql.planner.plan.TableFinishNode in project hetu-core by openlookeng.
the class LogicalPlanner method createDeletePlan.
private RelationPlan createDeletePlan(Analysis analysis, Delete node) {
TableHandle handle = analysis.getTableHandle(node.getTable());
if (handle.getConnectorHandle().isDeleteAsInsertSupported()) {
QueryPlanner.UpdateDeleteRelationPlan deletePlan = new QueryPlanner(analysis, planSymbolAllocator, idAllocator, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator), metadata, session, namedSubPlan, uniqueIdAllocator).planDeleteRowAsInsert(node);
RelationPlan plan = deletePlan.getPlan();
Optional<NewTableLayout> newTableLayout = metadata.getUpdateLayout(session, handle);
TableMetadata tableMetadata = metadata.getTableMetadata(session, handle);
String catalogName = handle.getCatalogName().getCatalogName();
Optional<Expression> constraint = deletePlan.getPredicate().isPresent() ? Optional.of(OriginalExpressionUtils.castToExpression(deletePlan.getPredicate().get())) : Optional.empty();
// Skip statistics collection for delete,
// because stats collection for delete seems to corrupt existing statistics
TableStatisticsMetadata statisticsMetadata = TableStatisticsMetadata.empty();
return createTableWriterPlan(analysis, plan, new TableWriterNode.DeleteAsInsertReference(handle, constraint, deletePlan.getColumnAssignments()), deletePlan.getColumNames(), newTableLayout, statisticsMetadata);
} else {
DeleteNode deleteNode = new QueryPlanner(analysis, planSymbolAllocator, idAllocator, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator), metadata, session, namedSubPlan, uniqueIdAllocator).plan(node);
TableFinishNode commitNode = new TableFinishNode(idAllocator.getNextId(), deleteNode, deleteNode.getTarget(), planSymbolAllocator.newSymbol("rows", BIGINT), Optional.empty(), Optional.empty());
return new RelationPlan(commitNode, analysis.getScope(node), commitNode.getOutputSymbols());
}
}
use of io.prestosql.sql.planner.plan.TableFinishNode in project hetu-core by openlookeng.
the class LogicalPlanner method createVacuumWriterPlan.
private RelationPlan createVacuumWriterPlan(Analysis analysis, TableHandle handle, VacuumTable node, WriterTarget target, List<Symbol> symbols, List<String> columnNames, TableStatisticsMetadata statisticsMetadata) {
Optional<StatisticAggregations> statisticsAggregation = Optional.empty();
Optional<StatisticAggregationsDescriptor<Symbol>> statisticsAggregationDescriptor = Optional.empty();
Optional<StatisticAggregations> finalStatisticsAggregation = Optional.empty();
Optional<StatisticAggregationsDescriptor<Symbol>> finalStatisticsAggregationDescriptor = Optional.empty();
if (!statisticsMetadata.isEmpty()) {
verify(columnNames.size() == symbols.size(), "columnNames.size() != symbols.size(): %s and %s", columnNames, symbols);
Map<String, Symbol> columnToSymbolMap = zip(columnNames.stream(), symbols.stream(), SimpleImmutableEntry::new).collect(toImmutableMap(Entry::getKey, Entry::getValue));
TableStatisticAggregation result = statisticsAggregationPlanner.createStatisticsAggregation(statisticsMetadata, columnToSymbolMap);
StatisticAggregations.Parts aggregations = result.getAggregations().createPartialAggregations(planSymbolAllocator, metadata);
// partial aggregation is run within the VacuumTableOperator to calculate the statistics for
// the data consumed by the VacuumTableOperator
// final aggregation is run within the TableFinishOperator to summarize collected statistics
// by the partial aggregation from all of the writer nodes
statisticsAggregation = Optional.of(aggregations.getPartialAggregation());
statisticsAggregationDescriptor = Optional.of(result.getDescriptor().map(aggregations.getMappings()::get));
finalStatisticsAggregation = Optional.of(aggregations.getFinalAggregation());
finalStatisticsAggregationDescriptor = Optional.of(result.getDescriptor());
}
VacuumTableNode vacuumTableNode = new VacuumTableNode(idAllocator.getNextId(), handle, target, planSymbolAllocator.newSymbol("partialrows", BIGINT), planSymbolAllocator.newSymbol("fragment", VARBINARY), node.getPartition().orElse(""), node.isFull(), symbols, statisticsAggregation, statisticsAggregationDescriptor);
TableFinishNode commitNode = new TableFinishNode(idAllocator.getNextId(), vacuumTableNode, target, planSymbolAllocator.newSymbol("rows", BIGINT), finalStatisticsAggregation, finalStatisticsAggregationDescriptor);
return new RelationPlan(commitNode, analysis.getRootScope(), commitNode.getOutputSymbols());
}
use of io.prestosql.sql.planner.plan.TableFinishNode in project hetu-core by openlookeng.
the class LogicalPlanner method createUpdatePlan.
private RelationPlan createUpdatePlan(Analysis analysis, Update node) {
TableHandle handle = analysis.getTableHandle(node.getTable());
if (handle.getConnectorHandle().isUpdateAsInsertSupported()) {
QueryPlanner.UpdateDeleteRelationPlan updatePlan = new QueryPlanner(analysis, planSymbolAllocator, idAllocator, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator), metadata, session, namedSubPlan, uniqueIdAllocator).planUpdateRowAsInsert(node);
RelationPlan plan = updatePlan.getPlan();
Analysis.Update update = analysis.getUpdate().get();
TableMetadata tableMetadata = metadata.getTableMetadata(session, update.getTarget());
Optional<NewTableLayout> newTableLayout = metadata.getUpdateLayout(session, update.getTarget());
String catalogName = update.getTarget().getCatalogName().getCatalogName();
TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, catalogName, tableMetadata.getMetadata());
Optional<Expression> constraint = updatePlan.getPredicate().isPresent() ? Optional.of(OriginalExpressionUtils.castToExpression(updatePlan.getPredicate().get())) : Optional.empty();
return createTableWriterPlan(analysis, plan, new TableWriterNode.UpdateReference(update.getTarget(), constraint, updatePlan.getColumnAssignments()), updatePlan.getColumNames(), newTableLayout, statisticsMetadata);
}
UpdateNode updateNode = new QueryPlanner(analysis, planSymbolAllocator, idAllocator, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator), metadata, session, namedSubPlan, uniqueIdAllocator).plan(node);
TableFinishNode commitNode = new TableFinishNode(idAllocator.getNextId(), updateNode, updateNode.getTarget(), planSymbolAllocator.newSymbol("rows", BIGINT), Optional.empty(), Optional.empty());
return new RelationPlan(commitNode, analysis.getScope(node), commitNode.getOutputSymbols());
}
Aggregations