use of io.prestosql.spi.plan.AggregationNode in project hetu-core by openlookeng.
the class RewriteSpatialPartitioningAggregation method apply.
@Override
public Result apply(AggregationNode node, Captures captures, Context context) {
ImmutableMap.Builder<Symbol, Aggregation> aggregations = ImmutableMap.builder();
Symbol partitionCountSymbol = context.getSymbolAllocator().newSymbol("partition_count", INTEGER);
ImmutableMap.Builder<Symbol, RowExpression> envelopeAssignments = ImmutableMap.builder();
for (Map.Entry<Symbol, Aggregation> entry : node.getAggregations().entrySet()) {
Aggregation aggregation = entry.getValue();
QualifiedObjectName name = metadata.getFunctionAndTypeManager().getFunctionMetadata(aggregation.getFunctionHandle()).getName();
Type geometryType = metadata.getType(GEOMETRY_TYPE_SIGNATURE);
if (name.equals(NAME) && aggregation.getArguments().size() == 1) {
RowExpression geometry = getOnlyElement(aggregation.getArguments().stream().collect(toImmutableList()));
Symbol envelopeSymbol = context.getSymbolAllocator().newSymbol("envelope", metadata.getType(GEOMETRY_TYPE_SIGNATURE));
if (isFunctionNameMatch(geometry, "ST_Envelope")) {
envelopeAssignments.put(envelopeSymbol, geometry);
} else {
envelopeAssignments.put(envelopeSymbol, castToRowExpression(new FunctionCallBuilder(metadata).setName(QualifiedName.of("ST_Envelope")).addArgument(GEOMETRY_TYPE_SIGNATURE, castToExpression(geometry)).build()));
}
aggregations.put(entry.getKey(), new Aggregation(new CallExpression(name.getObjectName(), metadata.getFunctionAndTypeManager().lookupFunction(NAME.getObjectName(), fromTypes(geometryType, INTEGER)), context.getSymbolAllocator().getTypes().get(entry.getKey()), ImmutableList.of(castToRowExpression(toSymbolReference(envelopeSymbol)), castToRowExpression(toSymbolReference(partitionCountSymbol))), Optional.empty()), ImmutableList.of(castToRowExpression(toSymbolReference(envelopeSymbol)), castToRowExpression(toSymbolReference(partitionCountSymbol))), false, Optional.empty(), Optional.empty(), aggregation.getMask()));
} else {
aggregations.put(entry);
}
}
return Result.ofPlanNode(new AggregationNode(node.getId(), new ProjectNode(context.getIdAllocator().getNextId(), node.getSource(), Assignments.builder().putAll(AssignmentUtils.identityAsSymbolReferences(node.getSource().getOutputSymbols())).put(partitionCountSymbol, castToRowExpression(new LongLiteral(Integer.toString(getHashPartitionCount(context.getSession()))))).putAll(envelopeAssignments.build()).build()), aggregations.build(), node.getGroupingSets(), node.getPreGroupedSymbols(), node.getStep(), node.getHashSymbol(), node.getGroupIdSymbol(), node.getAggregationType(), node.getFinalizeSymbol()));
}
use of io.prestosql.spi.plan.AggregationNode in project hetu-core by openlookeng.
the class StarTreeAggregationRule method optimize.
public Result optimize(AggregationNode aggregationNode, final PlanNode filterNode, TableScanNode tableScanNode, Map<String, Object> symbolMapping, Session session, PlanSymbolAllocator symbolAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) {
TableHandle tableHandle = tableScanNode.getTable();
TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle);
String tableName = tableMetadata.getQualifiedName().toString();
CubeStatement statement = CubeStatementGenerator.generate(tableName, aggregationNode, symbolMapping);
// Don't use star-tree for non-aggregate queries
if (statement.getAggregations().isEmpty()) {
return Result.empty();
}
boolean hasDistinct = statement.getAggregations().stream().anyMatch(AggregationSignature::isDistinct);
// Since cube is pre-aggregated, utilising it for such queries could return incorrect result
if (aggregationNode.hasEmptyGroupingSet() && hasDistinct) {
return Result.empty();
}
List<CubeMetadata> cubeMetadataList = CubeMetadata.filter(this.cubeMetaStore.getMetadataList(statement.getFrom()), statement);
// Compare FilterNode predicate with Cube predicates to evaluate which cube can be used.
List<CubeMetadata> matchedCubeMetadataList = cubeMetadataList.stream().filter(cubeMetadata -> filterPredicateMatches((FilterNode) filterNode, cubeMetadata, session, symbolAllocator.getTypes())).collect(Collectors.toList());
// Match based on filter conditions
if (matchedCubeMetadataList.isEmpty()) {
return Result.empty();
}
LongSupplier lastModifiedTimeSupplier = metadata.getTableLastModifiedTimeSupplier(session, tableHandle);
if (lastModifiedTimeSupplier == null) {
warningCollector.add(new PrestoWarning(EXPIRED_CUBE, "Unable to identify last modified time of " + tableName + ". Ignoring star tree cubes."));
return Result.empty();
}
// Filter out cubes that were created before the source table was updated
long lastModifiedTime = lastModifiedTimeSupplier.getAsLong();
// There was a problem retrieving last modified time, we should skip using star tree rather than failing the query
if (lastModifiedTime == -1L) {
return Result.empty();
}
matchedCubeMetadataList = matchedCubeMetadataList.stream().filter(cubeMetadata -> cubeMetadata.getSourceTableLastUpdatedTime() >= lastModifiedTime).collect(Collectors.toList());
if (matchedCubeMetadataList.isEmpty()) {
warningCollector.add(new PrestoWarning(EXPIRED_CUBE, tableName + " has been modified after creating cubes. Ignoring expired cubes."));
return Result.empty();
}
// If multiple cubes are matching then lets select the recent built cube
// so sort the cube based on the last updated time stamp
matchedCubeMetadataList.sort(Comparator.comparingLong(CubeMetadata::getLastUpdatedTime).reversed());
CubeMetadata matchedCubeMetadata = matchedCubeMetadataList.get(0);
AggregationRewriteWithCube aggregationRewriteWithCube = new AggregationRewriteWithCube(metadata, session, symbolAllocator, idAllocator, symbolMapping, matchedCubeMetadata);
return Result.ofPlanNode(aggregationRewriteWithCube.rewrite(aggregationNode, rewriteByRemovingSourceFilter(filterNode, matchedCubeMetadata)));
}
use of io.prestosql.spi.plan.AggregationNode in project hetu-core by openlookeng.
the class ScalarAggregationToJoinRewriter method createAggregationNode.
private Optional<AggregationNode> createAggregationNode(AggregationNode scalarAggregation, JoinNode leftOuterJoin, Symbol nonNullableAggregationSourceSymbol) {
ImmutableMap.Builder<Symbol, Aggregation> aggregations = ImmutableMap.builder();
for (Map.Entry<Symbol, Aggregation> entry : scalarAggregation.getAggregations().entrySet()) {
Aggregation aggregation = entry.getValue();
Symbol symbol = entry.getKey();
if (functionResolution.isCountFunction(entry.getValue().getFunctionHandle())) {
aggregations.put(symbol, new Aggregation(new CallExpression("count", functionResolution.countFunction(planSymbolAllocator.getTypes().get(nonNullableAggregationSourceSymbol)), BIGINT, ImmutableList.of(castToRowExpression(toSymbolReference(nonNullableAggregationSourceSymbol)))), ImmutableList.of(castToRowExpression(toSymbolReference(nonNullableAggregationSourceSymbol))), false, Optional.empty(), Optional.empty(), aggregation.getMask()));
} else {
aggregations.put(symbol, aggregation);
}
}
return Optional.of(new AggregationNode(idAllocator.getNextId(), leftOuterJoin, aggregations.build(), singleGroupingSet(leftOuterJoin.getLeft().getOutputSymbols()), ImmutableList.of(), scalarAggregation.getStep(), scalarAggregation.getHashSymbol(), Optional.empty(), scalarAggregation.getAggregationType(), scalarAggregation.getFinalizeSymbol()));
}
use of io.prestosql.spi.plan.AggregationNode in project boostkit-bigdata by kunpengcompute.
the class TestHivePartialAggregationPushdown method testPartialAggregationAndProjectPushdown.
@Test
public void testPartialAggregationAndProjectPushdown() {
// select count(x + 5) from table group by x
TableScanNode tableScanNode = buildTableScanNode(COLUMN_INT);
CallExpression callExpression = createOperationExpression(OperatorType.ADD, new VariableReferenceExpression(COLUMN_INT.getName(), INTEGER), new ConstantExpression(5, INTEGER));
List<Symbol> symbols = ImmutableList.of(new Symbol(COLUMN_INT.getName()));
List<RowExpression> rowExpressions = ImmutableList.of(callExpression);
ProjectNode projectNode = buildProjectNode(tableScanNode, symbols, rowExpressions);
AggregationNode aggregationNode = buildCountAggregationNode(projectNode);
PlanNode output = AGGREGATION_OPTIMIZER.optimize(aggregationNode, OFFLOAD_SESSION, COLUMN_TYPE_MAP, SYMBOL_ALLOCATOR, ID_ALLOCATOR);
Assignments assignmentsExpected = buildAssignments(symbols, rowExpressions);
matchProjection(output, assignmentsExpected.getMap());
}
use of io.prestosql.spi.plan.AggregationNode in project hetu-core by openlookeng.
the class PushAggregationThroughOuterJoin method createAggregationOverNull.
private Optional<MappedAggregationInfo> createAggregationOverNull(AggregationNode referenceAggregation, PlanSymbolAllocator planSymbolAllocator, PlanNodeIdAllocator idAllocator, Lookup lookup) {
// Create a values node that consists of a single row of nulls.
// Map the output symbols from the referenceAggregation's source
// to symbol references for the new values node.
ImmutableList.Builder<Symbol> nullSymbols = ImmutableList.builder();
ImmutableList.Builder<RowExpression> nullLiterals = ImmutableList.builder();
ImmutableMap.Builder<VariableReferenceExpression, VariableReferenceExpression> sourcesSymbolMappingBuilder = ImmutableMap.builder();
for (Symbol sourceSymbol : referenceAggregation.getSource().getOutputSymbols()) {
RowExpression nullLiteral = new ConstantExpression(null, planSymbolAllocator.getTypes().get(sourceSymbol));
nullLiterals.add(nullLiteral);
Symbol nullSymbol = planSymbolAllocator.newSymbol(nullLiteral);
nullSymbols.add(nullSymbol);
sourcesSymbolMappingBuilder.put(toVariableReference(sourceSymbol, planSymbolAllocator.getTypes().get(sourceSymbol)), toVariableReference(nullSymbol, nullLiteral.getType()));
}
ValuesNode nullRow = new ValuesNode(idAllocator.getNextId(), nullSymbols.build(), ImmutableList.of(nullLiterals.build()));
Map<VariableReferenceExpression, VariableReferenceExpression> sourcesSymbolMapping = sourcesSymbolMappingBuilder.build();
// For each aggregation function in the reference node, create a corresponding aggregation function
// that points to the nullRow. Map the symbols from the aggregations in referenceAggregation to the
// symbols in these new aggregations.
ImmutableMap.Builder<Symbol, Symbol> aggregationsSymbolMappingBuilder = ImmutableMap.builder();
ImmutableMap.Builder<Symbol, AggregationNode.Aggregation> aggregationsOverNullBuilder = ImmutableMap.builder();
for (Map.Entry<Symbol, AggregationNode.Aggregation> entry : referenceAggregation.getAggregations().entrySet()) {
Symbol aggregationSymbol = entry.getKey();
AggregationNode.Aggregation aggregation = entry.getValue();
if (!isUsingVariables(aggregation, sourcesSymbolMapping.keySet())) {
return Optional.empty();
}
Aggregation overNullAggregation = new Aggregation(new CallExpression(aggregation.getFunctionCall().getDisplayName(), aggregation.getFunctionCall().getFunctionHandle(), aggregation.getFunctionCall().getType(), aggregation.getArguments().stream().map(argument -> inlineVariables(sourcesSymbolMapping, argument)).collect(toImmutableList()), Optional.empty()), aggregation.getArguments().stream().map(argument -> inlineVariables(sourcesSymbolMapping, argument)).collect(toImmutableList()), aggregation.isDistinct(), aggregation.getFilter(), aggregation.getOrderingScheme(), aggregation.getMask());
Symbol overNullSymbol = planSymbolAllocator.newSymbol(overNullAggregation.getFunctionCall().getDisplayName(), planSymbolAllocator.getTypes().get(aggregationSymbol));
aggregationsOverNullBuilder.put(overNullSymbol, overNullAggregation);
aggregationsSymbolMappingBuilder.put(aggregationSymbol, overNullSymbol);
}
Map<Symbol, Symbol> aggregationsSymbolMapping = aggregationsSymbolMappingBuilder.build();
// create an aggregation node whose source is the null row.
AggregationNode aggregationOverNullRow = new AggregationNode(idAllocator.getNextId(), nullRow, aggregationsOverNullBuilder.build(), globalAggregation(), ImmutableList.of(), AggregationNode.Step.SINGLE, Optional.empty(), Optional.empty(), AggregationNode.AggregationType.HASH, Optional.empty());
return Optional.of(new MappedAggregationInfo(aggregationOverNullRow, aggregationsSymbolMapping));
}
Aggregations