use of io.prestosql.sql.planner.PlanSymbolAllocator in project hetu-core by openlookeng.
the class StarTreeAggregationRule method optimize.
public Result optimize(AggregationNode aggregationNode, final PlanNode filterNode, TableScanNode tableScanNode, Map<String, Object> symbolMapping, Session session, PlanSymbolAllocator symbolAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) {
TableHandle tableHandle = tableScanNode.getTable();
TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle);
String tableName = tableMetadata.getQualifiedName().toString();
CubeStatement statement = CubeStatementGenerator.generate(tableName, aggregationNode, symbolMapping);
// Don't use star-tree for non-aggregate queries
if (statement.getAggregations().isEmpty()) {
return Result.empty();
}
boolean hasDistinct = statement.getAggregations().stream().anyMatch(AggregationSignature::isDistinct);
// Since cube is pre-aggregated, utilising it for such queries could return incorrect result
if (aggregationNode.hasEmptyGroupingSet() && hasDistinct) {
return Result.empty();
}
List<CubeMetadata> cubeMetadataList = CubeMetadata.filter(this.cubeMetaStore.getMetadataList(statement.getFrom()), statement);
// Compare FilterNode predicate with Cube predicates to evaluate which cube can be used.
List<CubeMetadata> matchedCubeMetadataList = cubeMetadataList.stream().filter(cubeMetadata -> filterPredicateMatches((FilterNode) filterNode, cubeMetadata, session, symbolAllocator.getTypes())).collect(Collectors.toList());
// Match based on filter conditions
if (matchedCubeMetadataList.isEmpty()) {
return Result.empty();
}
LongSupplier lastModifiedTimeSupplier = metadata.getTableLastModifiedTimeSupplier(session, tableHandle);
if (lastModifiedTimeSupplier == null) {
warningCollector.add(new PrestoWarning(EXPIRED_CUBE, "Unable to identify last modified time of " + tableName + ". Ignoring star tree cubes."));
return Result.empty();
}
// Filter out cubes that were created before the source table was updated
long lastModifiedTime = lastModifiedTimeSupplier.getAsLong();
// There was a problem retrieving last modified time, we should skip using star tree rather than failing the query
if (lastModifiedTime == -1L) {
return Result.empty();
}
matchedCubeMetadataList = matchedCubeMetadataList.stream().filter(cubeMetadata -> cubeMetadata.getSourceTableLastUpdatedTime() >= lastModifiedTime).collect(Collectors.toList());
if (matchedCubeMetadataList.isEmpty()) {
warningCollector.add(new PrestoWarning(EXPIRED_CUBE, tableName + " has been modified after creating cubes. Ignoring expired cubes."));
return Result.empty();
}
// If multiple cubes are matching then lets select the recent built cube
// so sort the cube based on the last updated time stamp
matchedCubeMetadataList.sort(Comparator.comparingLong(CubeMetadata::getLastUpdatedTime).reversed());
CubeMetadata matchedCubeMetadata = matchedCubeMetadataList.get(0);
AggregationRewriteWithCube aggregationRewriteWithCube = new AggregationRewriteWithCube(metadata, session, symbolAllocator, idAllocator, symbolMapping, matchedCubeMetadata);
return Result.ofPlanNode(aggregationRewriteWithCube.rewrite(aggregationNode, rewriteByRemovingSourceFilter(filterNode, matchedCubeMetadata)));
}
use of io.prestosql.sql.planner.PlanSymbolAllocator in project boostkit-bigdata by kunpengcompute.
the class TestHiveFilterPushdown method testFilterPushdownWithAlias.
@Test
public void testFilterPushdownWithAlias() {
Map<Symbol, Type> initialSymbolsMap = new HashMap<>(SYMBOL_ALLOCATOR.getSymbols());
Symbol oldSymbol = new Symbol(COLUMN_INT.getName());
assertTrue(initialSymbolsMap.containsKey(oldSymbol));
Type columnType = initialSymbolsMap.remove(oldSymbol);
String columnMappingName = COLUMN_INT.getName() + "_0";
initialSymbolsMap.put(new Symbol(columnMappingName), columnType);
PlanSymbolAllocator symbolAllocator = new PlanSymbolAllocator(initialSymbolsMap);
List<String> names = ImmutableList.of(columnMappingName);
List<HiveColumnHandle> columnHandles = ImmutableList.of(COLUMN_INT);
TableScanNode tableScanNode = buildTableScanNodeWithAlias(names, columnHandles);
String predicate = String.format("%s > 5", columnMappingName);
RowExpression expression = TRANSLATOR.translate(predicate, symbolAllocator.getSymbols());
FilterNode filterNode = buildFilterNode(tableScanNode, expression);
Map<String, Type> columnTypesMap = new HashMap<>(COLUMN_TYPE_MAP);
columnTypesMap.remove(COLUMN_INT.getName());
columnTypesMap.put(columnMappingName, columnType);
PlanNode node = FILTER_OPTIMIZER.optimize(filterNode, OFFLOAD_SESSION, columnTypesMap, symbolAllocator, ID_ALLOCATOR);
matchFilterOffloadWithAlias(node, "GREATER_THAN(_int, 5)");
}
use of io.prestosql.sql.planner.PlanSymbolAllocator in project hetu-core by openlookeng.
the class PushAggregationThroughOuterJoin method createAggregationOverNull.
private Optional<MappedAggregationInfo> createAggregationOverNull(AggregationNode referenceAggregation, PlanSymbolAllocator planSymbolAllocator, PlanNodeIdAllocator idAllocator, Lookup lookup) {
// Create a values node that consists of a single row of nulls.
// Map the output symbols from the referenceAggregation's source
// to symbol references for the new values node.
ImmutableList.Builder<Symbol> nullSymbols = ImmutableList.builder();
ImmutableList.Builder<RowExpression> nullLiterals = ImmutableList.builder();
ImmutableMap.Builder<VariableReferenceExpression, VariableReferenceExpression> sourcesSymbolMappingBuilder = ImmutableMap.builder();
for (Symbol sourceSymbol : referenceAggregation.getSource().getOutputSymbols()) {
RowExpression nullLiteral = new ConstantExpression(null, planSymbolAllocator.getTypes().get(sourceSymbol));
nullLiterals.add(nullLiteral);
Symbol nullSymbol = planSymbolAllocator.newSymbol(nullLiteral);
nullSymbols.add(nullSymbol);
sourcesSymbolMappingBuilder.put(toVariableReference(sourceSymbol, planSymbolAllocator.getTypes().get(sourceSymbol)), toVariableReference(nullSymbol, nullLiteral.getType()));
}
ValuesNode nullRow = new ValuesNode(idAllocator.getNextId(), nullSymbols.build(), ImmutableList.of(nullLiterals.build()));
Map<VariableReferenceExpression, VariableReferenceExpression> sourcesSymbolMapping = sourcesSymbolMappingBuilder.build();
// For each aggregation function in the reference node, create a corresponding aggregation function
// that points to the nullRow. Map the symbols from the aggregations in referenceAggregation to the
// symbols in these new aggregations.
ImmutableMap.Builder<Symbol, Symbol> aggregationsSymbolMappingBuilder = ImmutableMap.builder();
ImmutableMap.Builder<Symbol, AggregationNode.Aggregation> aggregationsOverNullBuilder = ImmutableMap.builder();
for (Map.Entry<Symbol, AggregationNode.Aggregation> entry : referenceAggregation.getAggregations().entrySet()) {
Symbol aggregationSymbol = entry.getKey();
AggregationNode.Aggregation aggregation = entry.getValue();
if (!isUsingVariables(aggregation, sourcesSymbolMapping.keySet())) {
return Optional.empty();
}
Aggregation overNullAggregation = new Aggregation(new CallExpression(aggregation.getFunctionCall().getDisplayName(), aggregation.getFunctionCall().getFunctionHandle(), aggregation.getFunctionCall().getType(), aggregation.getArguments().stream().map(argument -> inlineVariables(sourcesSymbolMapping, argument)).collect(toImmutableList()), Optional.empty()), aggregation.getArguments().stream().map(argument -> inlineVariables(sourcesSymbolMapping, argument)).collect(toImmutableList()), aggregation.isDistinct(), aggregation.getFilter(), aggregation.getOrderingScheme(), aggregation.getMask());
Symbol overNullSymbol = planSymbolAllocator.newSymbol(overNullAggregation.getFunctionCall().getDisplayName(), planSymbolAllocator.getTypes().get(aggregationSymbol));
aggregationsOverNullBuilder.put(overNullSymbol, overNullAggregation);
aggregationsSymbolMappingBuilder.put(aggregationSymbol, overNullSymbol);
}
Map<Symbol, Symbol> aggregationsSymbolMapping = aggregationsSymbolMappingBuilder.build();
// create an aggregation node whose source is the null row.
AggregationNode aggregationOverNullRow = new AggregationNode(idAllocator.getNextId(), nullRow, aggregationsOverNullBuilder.build(), globalAggregation(), ImmutableList.of(), AggregationNode.Step.SINGLE, Optional.empty(), Optional.empty(), AggregationNode.AggregationType.HASH, Optional.empty());
return Optional.of(new MappedAggregationInfo(aggregationOverNullRow, aggregationsSymbolMapping));
}
use of io.prestosql.sql.planner.PlanSymbolAllocator in project hetu-core by openlookeng.
the class TestStarTreeAggregationRule method setupBeforeClass.
@BeforeClass
public void setupBeforeClass() {
PlanSymbolAllocator symbolAllocator = new PlanSymbolAllocator();
columnOrderkey = symbolAllocator.newSymbol("orderkey", BIGINT);
columnOrderDate = symbolAllocator.newSymbol("orderdate", DATE);
columnCustkey = symbolAllocator.newSymbol("custkey", BIGINT);
columnTotalprice = symbolAllocator.newSymbol("totalprice", DOUBLE);
orderkeyHandle = new TpchColumnHandle("orderkey", BIGINT);
orderdateHandle = new TpchColumnHandle("orderdate", DATE);
custkeyHandle = new TpchColumnHandle("custkey", BIGINT);
totalpriceHandle = new TpchColumnHandle("totalprice", DOUBLE);
ColumnMetadata orderKeyColumnMetadata = new ColumnMetadata(orderkeyHandle.getColumnName(), orderkeyHandle.getType());
ColumnMetadata orderDateColumnMetadata = new ColumnMetadata(orderdateHandle.getColumnName(), orderdateHandle.getType());
ColumnMetadata custKeyColumnMetadata = new ColumnMetadata(custkeyHandle.getColumnName(), custkeyHandle.getType());
ColumnMetadata totalPriceColumnMetadata = new ColumnMetadata(totalpriceHandle.getColumnName(), totalpriceHandle.getType());
output = symbolAllocator.newSymbol("output", DOUBLE);
assignments = ImmutableMap.<Symbol, ColumnHandle>builder().put(columnOrderkey, orderkeyHandle).put(columnOrderDate, orderdateHandle).put(columnCustkey, custkeyHandle).put(columnTotalprice, totalpriceHandle).build();
TpchTableHandle orders = new TpchTableHandle("orders", 1.0);
ordersTableHandle = new TableHandle(tester().getCurrentConnectorId(), orders, TpchTransactionHandle.INSTANCE, Optional.of(new TpchTableLayoutHandle(orders, TupleDomain.all())));
baseTableScan = new TableScanNode(newId(), ordersTableHandle, ImmutableList.copyOf(assignments.keySet()), assignments, TupleDomain.all(), Optional.empty(), ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), 0, false);
QualifiedObjectName baseTableName = QualifiedObjectName.valueOf(baseTableScan.getTable().getFullyQualifiedName());
baseTableMetadata = new TableMetadata(ordersTableHandle.getCatalogName(), new ConnectorTableMetadata(new SchemaTableName(baseTableName.getSchemaName(), baseTableName.getObjectName()), Arrays.asList(orderKeyColumnMetadata, orderDateColumnMetadata, custKeyColumnMetadata, totalPriceColumnMetadata)));
columnCountAll = symbolAllocator.newSymbol("count_all", BIGINT);
columnSumTotalPrice = symbolAllocator.newSymbol("sum_totalprice", DOUBLE);
columnCountOrderKey = symbolAllocator.newSymbol("count_orderkey", BIGINT);
columnGroupingBitSet = symbolAllocator.newSymbol("grouping_bit_set", BIGINT);
cubeColumnCustKey = symbolAllocator.newSymbol("custkey", BIGINT);
cubeColumnOrderDate = symbolAllocator.newSymbol("orderdate", DATE);
countAllHandle = new TpchColumnHandle("count_all", BIGINT);
sumTotalPriceHandle = new TpchColumnHandle("sum_totalprice", DOUBLE);
countOrderKeyHandle = new TpchColumnHandle("count_orderkey", BIGINT);
groupingBitSetHandle = new TpchColumnHandle("grouping_bit_set", BIGINT);
custKeyCubeColumnHandle = new TpchColumnHandle("custkey", BIGINT);
orderDateCubeColumnHandle = new TpchColumnHandle("orderdate", DATE);
ordersCubeColumnHandles.put(countAllHandle.getColumnName(), countAllHandle);
ordersCubeColumnHandles.put(sumTotalPriceHandle.getColumnName(), sumTotalPriceHandle);
ordersCubeColumnHandles.put(countOrderKeyHandle.getColumnName(), countOrderKeyHandle);
ordersCubeColumnHandles.put(groupingBitSetHandle.getColumnName(), groupingBitSetHandle);
ordersCubeColumnHandles.put(custKeyCubeColumnHandle.getColumnName(), custKeyCubeColumnHandle);
ordersCubeColumnHandles.put(orderDateCubeColumnHandle.getColumnName(), orderDateCubeColumnHandle);
TpchTableHandle ordersCube = new TpchTableHandle("orders_cube", 1.0);
ordersCubeHandle = new TableHandle(tester().getCurrentConnectorId(), ordersCube, TpchTransactionHandle.INSTANCE, Optional.of(new TpchTableLayoutHandle(ordersCube, TupleDomain.all())));
countAllColumnMetadata = new ColumnMetadata(countAllHandle.getColumnName(), countAllHandle.getType());
sumTotalPriceColumnMetadata = new ColumnMetadata(sumTotalPriceHandle.getColumnName(), sumTotalPriceHandle.getType());
countOrderKeyColumnMetadata = new ColumnMetadata(countOrderKeyHandle.getColumnName(), countOrderKeyHandle.getType());
groupingBitSetColumnMetadata = new ColumnMetadata(groupingBitSetHandle.getColumnName(), groupingBitSetHandle.getType());
custKeyCubeColumnMetadata = new ColumnMetadata(custKeyCubeColumnHandle.getColumnName(), custKeyCubeColumnHandle.getType());
orderDateCubeColumnMetadata = new ColumnMetadata(orderDateCubeColumnHandle.getColumnName(), orderDateCubeColumnHandle.getType());
config = new FeaturesConfig();
config.setEnableStarTreeIndex(true);
cubeManager = Mockito.mock(CubeManager.class);
provider = Mockito.mock(CubeProvider.class);
cubeMetaStore = Mockito.mock(CubeMetaStore.class);
cubeMetadata = Mockito.mock(CubeMetadata.class);
ordersTableHandleMatcher = new BaseMatcher<TableHandle>() {
@Override
public boolean matches(Object o) {
if (!(o instanceof TableHandle)) {
return false;
}
TableHandle th = (TableHandle) o;
return th.getFullyQualifiedName().equals(ordersTableHandle.getFullyQualifiedName());
}
@Override
public void describeTo(Description description) {
}
};
ordersCubeHandleMatcher = new BaseMatcher<TableHandle>() {
@Override
public boolean matches(Object o) {
if (!(o instanceof TableHandle)) {
return false;
}
TableHandle th = (TableHandle) o;
return th.getFullyQualifiedName().equals(ordersCubeHandle.getFullyQualifiedName());
}
@Override
public void describeTo(Description description) {
}
};
countAllColumnHandleMatcher = new BaseMatcher<ColumnHandle>() {
@Override
public void describeTo(Description description) {
}
@Override
public boolean matches(Object o) {
if (!(o instanceof ColumnHandle)) {
return false;
}
ColumnHandle ch = (ColumnHandle) o;
return ch.getColumnName().equalsIgnoreCase(countAllHandle.getColumnName());
}
};
sumTotalPriceColumnHandleMatcher = new BaseMatcher<ColumnHandle>() {
@Override
public void describeTo(Description description) {
}
@Override
public boolean matches(Object o) {
if (!(o instanceof ColumnHandle)) {
return false;
}
ColumnHandle ch = (ColumnHandle) o;
return ch.getColumnName().equalsIgnoreCase(sumTotalPriceHandle.getColumnName());
}
};
countOrderKeyColumnHandleMatcher = new BaseMatcher<ColumnHandle>() {
@Override
public boolean matches(Object o) {
if (!(o instanceof ColumnHandle)) {
return false;
}
ColumnHandle ch = (ColumnHandle) o;
return ch.getColumnName().equalsIgnoreCase(countOrderKeyHandle.getColumnName());
}
@Override
public void describeTo(Description description) {
}
};
groupingBitSetColumnHandleMatcher = new BaseMatcher<ColumnHandle>() {
@Override
public boolean matches(Object o) {
if (!(o instanceof ColumnHandle)) {
return false;
}
ColumnHandle ch = (ColumnHandle) o;
return ch.getColumnName().equalsIgnoreCase(groupingBitSetHandle.getColumnName());
}
@Override
public void describeTo(Description description) {
}
};
orderDateCubeColumnHandleMatcher = new BaseMatcher<ColumnHandle>() {
@Override
public boolean matches(Object o) {
if (!(o instanceof ColumnHandle)) {
return false;
}
ColumnHandle ch = (ColumnHandle) o;
return ch.getColumnName().equalsIgnoreCase(orderDateCubeColumnHandle.getColumnName());
}
@Override
public void describeTo(Description description) {
}
};
custKeyCubeColumnHandleMatcher = new BaseMatcher<ColumnHandle>() {
@Override
public boolean matches(Object o) {
if (!(o instanceof ColumnHandle)) {
return false;
}
ColumnHandle ch = (ColumnHandle) o;
return ch.getColumnName().equalsIgnoreCase(custKeyCubeColumnHandle.getColumnName());
}
@Override
public void describeTo(Description description) {
}
};
}
use of io.prestosql.sql.planner.PlanSymbolAllocator in project hetu-core by openlookeng.
the class TestStatisticAggregationsDescriptor method createTestDescriptor.
private static StatisticAggregationsDescriptor<Symbol> createTestDescriptor() {
StatisticAggregationsDescriptor.Builder<Symbol> builder = StatisticAggregationsDescriptor.builder();
PlanSymbolAllocator planSymbolAllocator = new PlanSymbolAllocator();
for (String column : COLUMNS) {
for (ColumnStatisticType type : ColumnStatisticType.values()) {
builder.addColumnStatistic(new ColumnStatisticMetadata(column, type), testSymbol(planSymbolAllocator));
}
builder.addGrouping(column, testSymbol(planSymbolAllocator));
}
builder.addTableStatistic(ROW_COUNT, testSymbol(planSymbolAllocator));
return builder.build();
}
Aggregations