Examples with CubeStatement io.hetu.core.spi.cube.CubeStatement used on opensource projects

Example 1 with CubeStatement

use of io.hetu.core.spi.cube.CubeStatement in project hetu-core by openlookeng.

the class CubeStatementGenerator method generate.

public static CubeStatement generate(String fromTable, AggregationNode aggregationNode, Map<String, Object> symbolMappings) {
    CubeStatement.Builder builder = CubeStatement.newBuilder();
    builder.from(fromTable);
    Map<Symbol, AggregationNode.Aggregation> aggregations = aggregationNode.getAggregations();
    // Extract selection and aggregation
    for (Symbol symbol : aggregationNode.getOutputSymbols()) {
        Object column = symbolMappings.get(symbol.getName());
        if (column instanceof ColumnHandle) {
            // Selection
            String columnName = ((ColumnHandle) column).getColumnName();
            builder.select(columnName);
        } else if (aggregations.containsKey(symbol)) {
            // Aggregation
            Map<Symbol, AggregationSignature> signatures = Collections.emptyMap();
            AggregationNode.Aggregation aggregation = aggregations.get(symbol);
            List<RowExpression> arguments = aggregation.getArguments();
            if (arguments.isEmpty()) {
                signatures = createSignature(aggregation, symbol, null);
            } else if (arguments.size() == 1) {
                RowExpression argument = arguments.get(0);
                if (OriginalExpressionUtils.isExpression(argument)) {
                    Expression argAsExpr = OriginalExpressionUtils.castToExpression(argument);
                    if (argAsExpr instanceof SymbolReference) {
                        signatures = createSignature(aggregation, symbol, symbolMappings.get(((SymbolReference) argAsExpr).getName()));
                    }
                } else if (argument instanceof VariableReferenceExpression) {
                    signatures = createSignature(aggregation, symbol, symbolMappings.get(((VariableReferenceExpression) argument).getName()));
                }
            }
            if (signatures.isEmpty()) {
                throw new IllegalArgumentException("Failed to generate aggregator signature");
            }
            for (Map.Entry<Symbol, AggregationSignature> aggEntry : signatures.entrySet()) {
                builder.aggregate(aggEntry.getValue());
            }
        } else {
            throw new IllegalArgumentException("Column " + column + " is not an actual column or expressions");
        }
    }
    // Extract group by
    for (Symbol symbol : aggregationNode.getGroupingKeys()) {
        Object column = symbolMappings.get(symbol.getName());
        if (column instanceof ColumnHandle) {
            builder.groupByAddString(((ColumnHandle) column).getColumnName());
        } else {
            // Don't know how to handle it
            throw new IllegalArgumentException("Column " + symbol + " is not an actual column");
        }
    }
    return builder.build();
}

Example 2 with CubeStatement

use of io.hetu.core.spi.cube.CubeStatement in project hetu-core by openlookeng.

the class TestStarTreeMetadata method testMetadataNotMatchesCubeStatement.

@Test
public void testMetadataNotMatchesCubeStatement() {
    CubeStatement statement = CubeStatement.newBuilder().select("returnflag", "linestatus").aggregate(avg("unknown", false)).from("tpch.tiny.lineitem").groupByAddStringList("returnflag", "linestatus").build();
    assertFalse(metadata.matches(statement), "failed to detect an invalid cube statement");
}

Example 3 with CubeStatement

use of io.hetu.core.spi.cube.CubeStatement in project hetu-core by openlookeng.

the class TestStarTreeMetadata method testMatchEmptyGroupMetadata.

@Test
public void testMatchEmptyGroupMetadata() {
    CubeStatement statement = CubeStatement.newBuilder().select("returnflag", "linestatus").aggregate(avg("quantity", false)).from("tpch.tiny.lineitem").build();
    assertTrue(emptyGroupMetadata.matches(statement));
}

Example 4 with CubeStatement

use of io.hetu.core.spi.cube.CubeStatement in project hetu-core by openlookeng.

the class StarTreeAggregationRule method optimize.

public Result optimize(AggregationNode aggregationNode, final PlanNode filterNode, TableScanNode tableScanNode, Map<String, Object> symbolMapping, Session session, PlanSymbolAllocator symbolAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) {
    TableHandle tableHandle = tableScanNode.getTable();
    TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle);
    String tableName = tableMetadata.getQualifiedName().toString();
    CubeStatement statement = CubeStatementGenerator.generate(tableName, aggregationNode, symbolMapping);
    // Don't use star-tree for non-aggregate queries
    if (statement.getAggregations().isEmpty()) {
        return Result.empty();
    }
    boolean hasDistinct = statement.getAggregations().stream().anyMatch(AggregationSignature::isDistinct);
    // Since cube is pre-aggregated, utilising it for such queries could return incorrect result
    if (aggregationNode.hasEmptyGroupingSet() && hasDistinct) {
        return Result.empty();
    }
    List<CubeMetadata> cubeMetadataList = CubeMetadata.filter(this.cubeMetaStore.getMetadataList(statement.getFrom()), statement);
    // Compare FilterNode predicate with Cube predicates to evaluate which cube can be used.
    List<CubeMetadata> matchedCubeMetadataList = cubeMetadataList.stream().filter(cubeMetadata -> filterPredicateMatches((FilterNode) filterNode, cubeMetadata, session, symbolAllocator.getTypes())).collect(Collectors.toList());
    // Match based on filter conditions
    if (matchedCubeMetadataList.isEmpty()) {
        return Result.empty();
    }
    LongSupplier lastModifiedTimeSupplier = metadata.getTableLastModifiedTimeSupplier(session, tableHandle);
    if (lastModifiedTimeSupplier == null) {
        warningCollector.add(new PrestoWarning(EXPIRED_CUBE, "Unable to identify last modified time of " + tableName + ". Ignoring star tree cubes."));
        return Result.empty();
    }
    // Filter out cubes that were created before the source table was updated
    long lastModifiedTime = lastModifiedTimeSupplier.getAsLong();
    // There was a problem retrieving last modified time, we should skip using star tree rather than failing the query
    if (lastModifiedTime == -1L) {
        return Result.empty();
    }
    matchedCubeMetadataList = matchedCubeMetadataList.stream().filter(cubeMetadata -> cubeMetadata.getSourceTableLastUpdatedTime() >= lastModifiedTime).collect(Collectors.toList());
    if (matchedCubeMetadataList.isEmpty()) {
        warningCollector.add(new PrestoWarning(EXPIRED_CUBE, tableName + " has been modified after creating cubes. Ignoring expired cubes."));
        return Result.empty();
    }
    // If multiple cubes are matching then lets select the recent built cube
    // so sort the cube based on the last updated time stamp
    matchedCubeMetadataList.sort(Comparator.comparingLong(CubeMetadata::getLastUpdatedTime).reversed());
    CubeMetadata matchedCubeMetadata = matchedCubeMetadataList.get(0);
    AggregationRewriteWithCube aggregationRewriteWithCube = new AggregationRewriteWithCube(metadata, session, symbolAllocator, idAllocator, symbolMapping, matchedCubeMetadata);
    return Result.ofPlanNode(aggregationRewriteWithCube.rewrite(aggregationNode, rewriteByRemovingSourceFilter(filterNode, matchedCubeMetadata)));
}

Example 5 with CubeStatement

use of io.hetu.core.spi.cube.CubeStatement in project hetu-core by openlookeng.

the class TestStarTreeMetadata method testMetadataMatchesCubeStatement.

@Test
public void testMetadataMatchesCubeStatement() {
    CubeStatement statement = CubeStatement.newBuilder().select("returnflag", "linestatus").aggregate(avg("quantity", false)).from("tpch.tiny.lineitem").groupByAddStringList("returnflag", "linestatus").build();
    assertTrue(metadata.matches(statement), "failed to match a valid cube statement");
}