use of io.prestosql.spi.statistics.TableStatisticsMetadata in project hetu-core by openlookeng.
the class LogicalPlanner method createInsertPlan.
private RelationPlan createInsertPlan(Analysis analysis, Insert insertStatement) {
Analysis.Insert insert = analysis.getInsert().get();
TableMetadata tableMetadata = metadata.getTableMetadata(session, insert.getTarget());
List<ColumnMetadata> visibleTableColumns = tableMetadata.getColumns().stream().filter(column -> !column.isHidden()).collect(toImmutableList());
List<String> visibleTableColumnNames = visibleTableColumns.stream().map(ColumnMetadata::getName).collect(toImmutableList());
RelationPlan plan = createRelationPlan(analysis, insertStatement.getQuery());
Map<String, ColumnHandle> columns = metadata.getColumnHandles(session, insert.getTarget());
Assignments.Builder assignments = Assignments.builder();
for (ColumnMetadata column : tableMetadata.getColumns()) {
if (column.isHidden()) {
continue;
}
Symbol output = planSymbolAllocator.newSymbol(column.getName(), column.getType());
int index = insert.getColumns().indexOf(columns.get(column.getName()));
if (index < 0) {
Expression cast = new Cast(new NullLiteral(), column.getType().getTypeSignature().toString());
assignments.put(output, castToRowExpression(cast));
} else {
Symbol input = plan.getSymbol(index);
Type tableType = column.getType();
Type queryType = planSymbolAllocator.getTypes().get(input);
if (queryType.equals(tableType) || typeCoercion.isTypeOnlyCoercion(queryType, tableType)) {
assignments.put(output, castToRowExpression(toSymbolReference(input)));
} else {
Expression cast = noTruncationCast(toSymbolReference(input), queryType, tableType);
assignments.put(output, castToRowExpression(cast));
}
}
}
ProjectNode projectNode = new ProjectNode(idAllocator.getNextId(), plan.getRoot(), assignments.build());
List<Field> fields = visibleTableColumns.stream().map(column -> Field.newUnqualified(column.getName(), column.getType())).collect(toImmutableList());
Scope scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(fields)).build();
plan = new RelationPlan(projectNode, scope, projectNode.getOutputSymbols());
Optional<NewTableLayout> newTableLayout = metadata.getInsertLayout(session, insert.getTarget());
String catalogName = insert.getTarget().getCatalogName().getCatalogName();
TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, catalogName, tableMetadata.getMetadata());
return createTableWriterPlan(analysis, plan, new InsertReference(insert.getTarget(), analysis.isInsertOverwrite()), visibleTableColumnNames, newTableLayout, statisticsMetadata);
}
use of io.prestosql.spi.statistics.TableStatisticsMetadata in project hetu-core by openlookeng.
the class LogicalPlanner method createInsertCubePlan.
private RelationPlan createInsertCubePlan(Analysis analysis, InsertCube insertCubeStatement) {
Analysis.CubeInsert insert = analysis.getCubeInsert().get();
TableMetadata tableMetadata = metadata.getTableMetadata(session, insert.getTarget());
List<ColumnMetadata> visibleTableColumns = tableMetadata.getColumns().stream().filter(column -> !column.isHidden()).collect(toImmutableList());
List<String> visibleTableColumnNames = visibleTableColumns.stream().map(ColumnMetadata::getName).collect(toImmutableList());
RelationPlan plan = createRelationPlan(analysis, insertCubeStatement.getQuery());
Map<String, ColumnHandle> columns = metadata.getColumnHandles(session, insert.getTarget());
Assignments.Builder assignments = Assignments.builder();
for (ColumnMetadata column : tableMetadata.getColumns()) {
if (column.isHidden()) {
continue;
}
Symbol output = planSymbolAllocator.newSymbol(column.getName(), column.getType());
int index = insert.getColumns().indexOf(columns.get(column.getName()));
if (index < 0) {
Expression cast = new Cast(new NullLiteral(), column.getType().getTypeSignature().toString());
assignments.put(output, castToRowExpression(cast));
} else {
Symbol input = plan.getSymbol(index);
Type tableType = column.getType();
Type queryType = planSymbolAllocator.getTypes().get(input);
if (queryType.equals(tableType) || typeCoercion.isTypeOnlyCoercion(queryType, tableType)) {
assignments.put(output, castToRowExpression(toSymbolReference(input)));
} else {
Expression cast = noTruncationCast(toSymbolReference(input), queryType, tableType);
assignments.put(output, castToRowExpression(cast));
}
}
}
ProjectNode projectNode = new ProjectNode(idAllocator.getNextId(), plan.getRoot(), assignments.build());
List<Field> fields = visibleTableColumns.stream().map(column -> Field.newUnqualified(column.getName(), column.getType())).collect(toImmutableList());
Scope scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(fields)).build();
plan = new RelationPlan(projectNode, scope, projectNode.getOutputSymbols());
Optional<NewTableLayout> newTableLayout = metadata.getInsertLayout(session, insert.getTarget());
String catalogName = insert.getTarget().getCatalogName().getCatalogName();
TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, catalogName, tableMetadata.getMetadata());
RelationPlan tableWriterPlan = createTableWriterPlan(analysis, plan, new InsertReference(insert.getTarget(), analysis.isCubeOverwrite()), visibleTableColumnNames, newTableLayout, statisticsMetadata);
Expression rewritten = null;
Set<Identifier> predicateColumns = new HashSet<>();
if (insertCubeStatement.getWhere().isPresent()) {
rewritten = new QueryPlanner(analysis, planSymbolAllocator, idAllocator, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator), metadata, session, namedSubPlan, uniqueIdAllocator).rewriteExpression(tableWriterPlan, insertCubeStatement.getWhere().get(), analysis, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator));
predicateColumns.addAll(ExpressionUtils.getIdentifiers(rewritten));
}
CubeMetadata cubeMetadata = insert.getMetadata();
if (!insertCubeStatement.isOverwrite() && !insertCubeStatement.getWhere().isPresent() && cubeMetadata.getCubeStatus() != CubeStatus.INACTIVE) {
// Means data some data was inserted before, but trying to insert entire dataset
throw new PrestoException(QUERY_REJECTED, "Cannot allow insert. Inserting entire dataset but cube already has partial data");
} else if (insertCubeStatement.getWhere().isPresent()) {
if (!canSupportPredicate(rewritten)) {
throw new PrestoException(QUERY_REJECTED, String.format("Cannot support predicate '%s'", ExpressionFormatter.formatExpression(rewritten, Optional.empty())));
}
if (!insertCubeStatement.isOverwrite() && arePredicatesOverlapping(rewritten, cubeMetadata)) {
throw new PrestoException(QUERY_REJECTED, String.format("Cannot allow insert. Cube already contains data for the given predicate '%s'", ExpressionFormatter.formatExpression(insertCubeStatement.getWhere().get(), Optional.empty())));
}
}
TableHandle sourceTableHandle = insert.getSourceTable();
// At this point it has been verified that source table has not been updated
// so insert into cube should be allowed
LongSupplier tableLastModifiedTimeSupplier = metadata.getTableLastModifiedTimeSupplier(session, sourceTableHandle);
checkState(tableLastModifiedTimeSupplier != null, "Table last modified time is null");
Map<Symbol, Type> predicateColumnsType = predicateColumns.stream().map(identifier -> new Symbol(identifier.getValue())).collect(Collectors.toMap(Function.identity(), symbol -> planSymbolAllocator.getTypes().get(symbol), (key1, ignored) -> key1));
CubeFinishNode cubeFinishNode = new CubeFinishNode(idAllocator.getNextId(), tableWriterPlan.getRoot(), planSymbolAllocator.newSymbol("rows", BIGINT), new CubeUpdateMetadata(tableMetadata.getQualifiedName().toString(), tableLastModifiedTimeSupplier.getAsLong(), rewritten != null ? ExpressionFormatter.formatExpression(rewritten, Optional.empty()) : null, insertCubeStatement.isOverwrite()), predicateColumnsType);
return new RelationPlan(cubeFinishNode, analysis.getScope(insertCubeStatement), cubeFinishNode.getOutputSymbols());
}
use of io.prestosql.spi.statistics.TableStatisticsMetadata in project hetu-core by openlookeng.
the class LogicalPlanner method createDeletePlan.
private RelationPlan createDeletePlan(Analysis analysis, Delete node) {
TableHandle handle = analysis.getTableHandle(node.getTable());
if (handle.getConnectorHandle().isDeleteAsInsertSupported()) {
QueryPlanner.UpdateDeleteRelationPlan deletePlan = new QueryPlanner(analysis, planSymbolAllocator, idAllocator, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator), metadata, session, namedSubPlan, uniqueIdAllocator).planDeleteRowAsInsert(node);
RelationPlan plan = deletePlan.getPlan();
Optional<NewTableLayout> newTableLayout = metadata.getUpdateLayout(session, handle);
TableMetadata tableMetadata = metadata.getTableMetadata(session, handle);
String catalogName = handle.getCatalogName().getCatalogName();
Optional<Expression> constraint = deletePlan.getPredicate().isPresent() ? Optional.of(OriginalExpressionUtils.castToExpression(deletePlan.getPredicate().get())) : Optional.empty();
// Skip statistics collection for delete,
// because stats collection for delete seems to corrupt existing statistics
TableStatisticsMetadata statisticsMetadata = TableStatisticsMetadata.empty();
return createTableWriterPlan(analysis, plan, new TableWriterNode.DeleteAsInsertReference(handle, constraint, deletePlan.getColumnAssignments()), deletePlan.getColumNames(), newTableLayout, statisticsMetadata);
} else {
DeleteNode deleteNode = new QueryPlanner(analysis, planSymbolAllocator, idAllocator, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator), metadata, session, namedSubPlan, uniqueIdAllocator).plan(node);
TableFinishNode commitNode = new TableFinishNode(idAllocator.getNextId(), deleteNode, deleteNode.getTarget(), planSymbolAllocator.newSymbol("rows", BIGINT), Optional.empty(), Optional.empty());
return new RelationPlan(commitNode, analysis.getScope(node), commitNode.getOutputSymbols());
}
}
use of io.prestosql.spi.statistics.TableStatisticsMetadata in project hetu-core by openlookeng.
the class LogicalPlanner method createAnalyzePlan.
private RelationPlan createAnalyzePlan(Analysis analysis, Analyze analyzeStatement) {
TableHandle targetTable = analysis.getAnalyzeTarget().get();
// Plan table scan
Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, targetTable);
ImmutableList.Builder<Symbol> tableScanOutputs = ImmutableList.builder();
ImmutableMap.Builder<Symbol, ColumnHandle> symbolToColumnHandle = ImmutableMap.builder();
ImmutableMap.Builder<String, Symbol> columnNameToSymbol = ImmutableMap.builder();
TableMetadata tableMetadata = metadata.getTableMetadata(session, targetTable);
for (ColumnMetadata column : tableMetadata.getColumns()) {
Symbol symbol = planSymbolAllocator.newSymbol(column.getName(), column.getType());
tableScanOutputs.add(symbol);
symbolToColumnHandle.put(symbol, columnHandles.get(column.getName()));
columnNameToSymbol.put(column.getName(), symbol);
}
TableStatisticsMetadata tableStatisticsMetadata = metadata.getStatisticsCollectionMetadata(session, targetTable.getCatalogName().getCatalogName(), tableMetadata.getMetadata());
TableStatisticAggregation tableStatisticAggregation = statisticsAggregationPlanner.createStatisticsAggregation(tableStatisticsMetadata, columnNameToSymbol.build());
StatisticAggregations statisticAggregations = tableStatisticAggregation.getAggregations();
List<Symbol> groupingSymbols = statisticAggregations.getGroupingSymbols();
PlanNode planNode = new StatisticsWriterNode(idAllocator.getNextId(), new AggregationNode(idAllocator.getNextId(), TableScanNode.newInstance(idAllocator.getNextId(), targetTable, tableScanOutputs.build(), symbolToColumnHandle.build(), ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), 0, false), statisticAggregations.getAggregations(), singleGroupingSet(groupingSymbols), ImmutableList.of(), AggregationNode.Step.SINGLE, Optional.empty(), Optional.empty(), AggregationNode.AggregationType.HASH, Optional.empty()), new StatisticsWriterNode.WriteStatisticsReference(targetTable), planSymbolAllocator.newSymbol("rows", BIGINT), tableStatisticsMetadata.getTableStatistics().contains(ROW_COUNT), tableStatisticAggregation.getDescriptor());
return new RelationPlan(planNode, analysis.getScope(analyzeStatement), planNode.getOutputSymbols());
}
use of io.prestosql.spi.statistics.TableStatisticsMetadata in project hetu-core by openlookeng.
the class LogicalPlanner method createTableCreationPlan.
private RelationPlan createTableCreationPlan(Analysis analysis, Query query) {
QualifiedObjectName destination = analysis.getCreateTableDestination().get();
RelationPlan plan = createRelationPlan(analysis, query);
ConnectorTableMetadata tableMetadata = createTableMetadata(destination, getOutputTableColumns(plan, analysis.getColumnAliases()), analysis.getCreateTableProperties(), analysis.getParameters(), analysis.getCreateTableComment());
analysis.setCreateTableMetadata(tableMetadata);
Optional<NewTableLayout> newTableLayout = metadata.getNewTableLayout(session, destination.getCatalogName(), tableMetadata);
List<String> columnNames = tableMetadata.getColumns().stream().filter(column -> !column.isHidden()).map(ColumnMetadata::getName).collect(toImmutableList());
TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, destination.getCatalogName(), tableMetadata);
return createTableWriterPlan(analysis, plan, new CreateReference(destination.getCatalogName(), tableMetadata, newTableLayout), columnNames, newTableLayout, statisticsMetadata);
}
Aggregations