Examples with CubeFilter io.hetu.core.spi.cube.CubeFilter used on opensource projects

Example 1 with CubeFilter

use of io.hetu.core.spi.cube.CubeFilter in project hetu-core by openlookeng.

the class StarTreeMetaStore method convertTablesToMetadata.

private List<CubeMetadata> convertTablesToMetadata(List<TableEntity> tableEntities) {
    List<CubeMetadata> cubeMetadataList = new ArrayList<>();
    tableEntities.forEach(table -> {
        List<ColumnEntity> cols = table.getColumns();
        StarTreeMetadataBuilder builder = new StarTreeMetadataBuilder(table.getParameters().get(STAR_TABLE_NAME), table.getParameters().get(SOURCE_TABLE_NAME));
        cols.forEach(col -> {
            if (col.getType().equals("aggregate")) {
                builder.addAggregationColumn(col.getName(), col.getParameters().get("aggregateFunction"), col.getParameters().get(ORIGINAL_COLUMN), Boolean.parseBoolean(col.getParameters().get("distinct")));
            } else if (col.getType().equals("dimension")) {
                builder.addDimensionColumn(col.getName(), col.getParameters().get(ORIGINAL_COLUMN));
            }
        });
        String groupingString = table.getParameters().get(GROUPING_STRING);
        // Create empty set to support Empty Group
        builder.addGroup((groupingString == null || groupingString.isEmpty()) ? new HashSet<>() : Sets.newHashSet(groupingString.split(COLUMN_DELIMITER)));
        String sourceTablePredicate = table.getParameters().get(SOURCE_FILTER_STRING);
        String cubePredicate = table.getParameters().get(PREDICATE_STRING);
        if (sourceTablePredicate != null || cubePredicate != null) {
            builder.withCubeFilter(new CubeFilter(sourceTablePredicate, cubePredicate));
        }
        builder.setCubeStatus(CubeStatus.forValue(Integer.parseInt(table.getParameters().get(CUBE_STATUS))));
        builder.setTableLastUpdatedTime(Long.parseLong(table.getParameters().get(SOURCE_TABLE_LAST_UPDATED_TIME)));
        builder.setCubeLastUpdatedTime(Long.parseLong(table.getParameters().get(CUBE_LAST_UPDATED_TIME)));
        cubeMetadataList.add(builder.build());
    });
    return cubeMetadataList;
}

Example 2 with CubeFilter

use of io.hetu.core.spi.cube.CubeFilter in project hetu-core by openlookeng.

the class CreateCubeTask method internalExecute.

@VisibleForTesting
public ListenableFuture<?> internalExecute(CreateCube statement, Metadata metadata, AccessControl accessControl, Session session, QueryStateMachine stateMachine, List<Expression> parameters) {
    Optional<CubeMetaStore> optionalCubeMetaStore = cubeManager.getMetaStore(STAR_TREE);
    if (!optionalCubeMetaStore.isPresent()) {
        throw new RuntimeException("HetuMetaStore is not initialized");
    }
    QualifiedObjectName cubeName = createQualifiedObjectName(session, statement, statement.getCubeName());
    QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getSourceTableName());
    Optional<TableHandle> cubeHandle = metadata.getTableHandle(session, cubeName);
    Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableName);
    if (optionalCubeMetaStore.get().getMetadataFromCubeName(cubeName.toString()).isPresent()) {
        if (!statement.isNotExists()) {
            throw new SemanticException(CUBE_ALREADY_EXISTS, statement, "Cube '%s' already exists", cubeName);
        }
        return immediateFuture(null);
    }
    if (cubeHandle.isPresent()) {
        if (!statement.isNotExists()) {
            throw new SemanticException(CUBE_OR_TABLE_ALREADY_EXISTS, statement, "Cube or Table '%s' already exists", cubeName);
        }
        return immediateFuture(null);
    }
    CatalogName catalogName = metadata.getCatalogHandle(session, cubeName.getCatalogName()).orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog not found: " + cubeName.getCatalogName()));
    if (!metadata.isPreAggregationSupported(session, catalogName)) {
        throw new PrestoException(StandardErrorCode.NOT_SUPPORTED, String.format("Cube cannot be created on catalog '%s'", catalogName.toString()));
    }
    if (!tableHandle.isPresent()) {
        throw new SemanticException(MISSING_TABLE, statement, "Table '%s' does not exist", tableName);
    }
    TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle.get());
    List<String> groupingSet = statement.getGroupingSet().stream().map(s -> s.getValue().toLowerCase(ENGLISH)).collect(Collectors.toList());
    Map<String, ColumnMetadata> sourceTableColumns = tableMetadata.getColumns().stream().collect(Collectors.toMap(ColumnMetadata::getName, col -> col));
    List<ColumnMetadata> cubeColumns = new ArrayList<>();
    Map<String, AggregationSignature> aggregations = new HashMap<>();
    Analysis analysis = analyzeStatement(statement, session, metadata, accessControl, parameters, stateMachine.getWarningCollector());
    Map<String, Field> fields = analysis.getOutputDescriptor().getAllFields().stream().collect(Collectors.toMap(col -> col.getName().map(String::toLowerCase).get(), col -> col));
    for (FunctionCall aggFunction : statement.getAggregations()) {
        String aggFunctionName = aggFunction.getName().toString().toLowerCase(ENGLISH);
        String argument = aggFunction.getArguments().isEmpty() || aggFunction.getArguments().get(0) instanceof LongLiteral ? null : ((Identifier) aggFunction.getArguments().get(0)).getValue().toLowerCase(ENGLISH);
        boolean distinct = aggFunction.isDistinct();
        String cubeColumnName = aggFunctionName + "_" + (argument == null ? "all" : argument) + (aggFunction.isDistinct() ? "_distinct" : "");
        CubeAggregateFunction cubeAggregateFunction = CubeAggregateFunction.valueOf(aggFunctionName.toUpperCase(ENGLISH));
        switch(cubeAggregateFunction) {
            case SUM:
                aggregations.put(cubeColumnName, AggregationSignature.sum(argument, distinct));
                break;
            case COUNT:
                AggregationSignature aggregationSignature = argument == null ? AggregationSignature.count() : AggregationSignature.count(argument, distinct);
                aggregations.put(cubeColumnName, aggregationSignature);
                break;
            case AVG:
                aggregations.put(cubeColumnName, AggregationSignature.avg(argument, distinct));
                break;
            case MAX:
                aggregations.put(cubeColumnName, AggregationSignature.max(argument, distinct));
                break;
            case MIN:
                aggregations.put(cubeColumnName, AggregationSignature.min(argument, distinct));
                break;
            default:
                throw new PrestoException(NOT_SUPPORTED, format("Unsupported aggregation function : %s", aggFunctionName));
        }
        Field tableField = fields.get(cubeColumnName);
        ColumnMetadata cubeCol = new ColumnMetadata(cubeColumnName, tableField.getType(), true, null, null, false, Collections.emptyMap());
        cubeColumns.add(cubeCol);
    }
    accessControl.checkCanCreateTable(session.getRequiredTransactionId(), session.getIdentity(), tableName);
    Map<String, Expression> sqlProperties = mapFromProperties(statement.getProperties());
    Map<String, Object> properties = metadata.getTablePropertyManager().getProperties(catalogName, cubeName.getCatalogName(), sqlProperties, session, metadata, parameters);
    if (properties.containsKey("partitioned_by")) {
        List<String> partitionCols = new ArrayList<>(((List<String>) properties.get("partitioned_by")));
        // put all partition columns at the end of the list
        groupingSet.removeAll(partitionCols);
        groupingSet.addAll(partitionCols);
    }
    for (String dimension : groupingSet) {
        if (!sourceTableColumns.containsKey(dimension)) {
            throw new SemanticException(MISSING_COLUMN, statement, "Column %s does not exist", dimension);
        }
        ColumnMetadata tableCol = sourceTableColumns.get(dimension);
        ColumnMetadata cubeCol = new ColumnMetadata(dimension, tableCol.getType(), tableCol.isNullable(), null, null, false, tableCol.getProperties());
        cubeColumns.add(cubeCol);
    }
    ConnectorTableMetadata cubeTableMetadata = new ConnectorTableMetadata(cubeName.asSchemaTableName(), ImmutableList.copyOf(cubeColumns), properties);
    try {
        metadata.createTable(session, cubeName.getCatalogName(), cubeTableMetadata, statement.isNotExists());
    } catch (PrestoException e) {
        // connectors are not required to handle the ignoreExisting flag
        if (!e.getErrorCode().equals(ALREADY_EXISTS.toErrorCode()) || !statement.isNotExists()) {
            throw e;
        }
    }
    CubeMetadataBuilder builder = optionalCubeMetaStore.get().getBuilder(cubeName.toString(), tableName.toString());
    groupingSet.forEach(dimension -> builder.addDimensionColumn(dimension, dimension));
    aggregations.forEach((column, aggregationSignature) -> builder.addAggregationColumn(column, aggregationSignature.getFunction(), aggregationSignature.getDimension(), aggregationSignature.isDistinct()));
    builder.addGroup(new HashSet<>(groupingSet));
    // Status and Table modified time will be updated on the first insert into the cube
    builder.setCubeStatus(CubeStatus.INACTIVE);
    builder.setTableLastUpdatedTime(-1L);
    statement.getSourceFilter().ifPresent(sourceTablePredicate -> {
        sourceTablePredicate = Coercer.addCoercions(sourceTablePredicate, analysis);
        builder.withCubeFilter(new CubeFilter(ExpressionFormatter.formatExpression(sourceTablePredicate, Optional.empty())));
    });
    builder.setCubeLastUpdatedTime(System.currentTimeMillis());
    optionalCubeMetaStore.get().persist(builder.build());
    return immediateFuture(null);
}

Example 3 with CubeFilter

use of io.hetu.core.spi.cube.CubeFilter in project hetu-core by openlookeng.

the class LogicalPlanner method arePredicatesOverlapping.

private boolean arePredicatesOverlapping(Expression inputNewDataPredicate, CubeMetadata cubeMetadata) {
    // Cannot do this check inside StatementAnalyzer because predicate expressions have not been rewritten by then.
    Expression newDataPredicate = inputNewDataPredicate;
    TypeProvider types = planSymbolAllocator.getTypes();
    newDataPredicate = ExpressionUtils.rewriteIdentifiersToSymbolReferences(newDataPredicate);
    ExpressionDomainTranslator.ExtractionResult decomposedNewDataPredicate = ExpressionDomainTranslator.fromPredicate(metadata, session, newDataPredicate, types);
    if (cubeMetadata.getCubeStatus() == CubeStatus.INACTIVE) {
        // Inactive cubes are empty. So inserts should be allowed.
        return false;
    }
    CubeFilter cubeFilter = cubeMetadata.getCubeFilter();
    if (cubeFilter == null || cubeFilter.getCubePredicate() == null) {
        // Means Cube was created for entire dataset.
        return true;
    }
    SqlParser sqlParser = new SqlParser();
    Expression cubePredicateAsExpr = sqlParser.createExpression(cubeFilter.getCubePredicate(), createParsingOptions(session));
    cubePredicateAsExpr = ExpressionUtils.rewriteIdentifiersToSymbolReferences(cubePredicateAsExpr);
    ExpressionDomainTranslator.ExtractionResult decomposedCubePredicate = ExpressionDomainTranslator.fromPredicate(metadata, session, cubePredicateAsExpr, types);
    return decomposedCubePredicate.getTupleDomain().overlaps(decomposedNewDataPredicate.getTupleDomain());
}

Example 4 with CubeFilter

use of io.hetu.core.spi.cube.CubeFilter in project hetu-core by openlookeng.

the class CubeOptimizer method filterPredicateMatches.

private boolean filterPredicateMatches(Expression queryPredicate, CubeMetadata cubeMetadata, Session session, TypeProvider types) {
    CubeFilter cubeFilter = cubeMetadata.getCubeFilter();
    if (cubeFilter == null) {
        // Cube was built for entire table
        return queryPredicate == null || doesCubeContainQueryPredicateColumns(queryPredicate, cubeMetadata);
    }
    if (queryPredicate == null) {
        // Query statement has no WHERE clause but CUBE was built for subset of original data
        return false;
    }
    SqlParser sqlParser = new SqlParser();
    Expression cubeSourceTablePredicate = cubeFilter.getSourceTablePredicate() == null ? null : sqlParser.createExpression(cubeFilter.getSourceTablePredicate(), new ParsingOptions());
    Pair<Expression, Expression> queryPredicateSplit = splitQueryPredicate(queryPredicate, cubeSourceTablePredicate);
    if (!arePredicatesEqual(queryPredicateSplit.getLeft(), cubeSourceTablePredicate, metadata, session, types)) {
        log.debug("Cube source table predicate %s not matching query predicate %s", cubeSourceTablePredicate, queryPredicate);
        return false;
    }
    // Check if columns in query predicate are all part of the Cube.
    if ((cubeFilter.getCubePredicate() != null && queryPredicateSplit.getRight() == null) || (queryPredicateSplit.getRight() != null && !doesCubeContainQueryPredicateColumns(queryPredicateSplit.getRight(), cubeMetadata))) {
        // Cube does not contain all columns in the remaining predicate
        return false;
    }
    if (cubeFilter.getCubePredicate() == null) {
        // Cube has no additional predicates to compare with. i.e. Cube can be used to optimize the query
        return true;
    }
    ExpressionDomainTranslator.ExtractionResult decomposedQueryPredicate = ExpressionDomainTranslator.fromPredicate(metadata, session, queryPredicateSplit.getRight(), types);
    if (!BooleanLiteral.TRUE_LITERAL.equals(decomposedQueryPredicate.getRemainingExpression())) {
        log.error("StarTree cube cannot support predicate %s", castToExpression(filterNode.getPredicate()));
        return false;
    }
    Expression cubePredicate = ExpressionUtils.rewriteIdentifiersToSymbolReferences(sqlParser.createExpression(cubeFilter.getCubePredicate(), new ParsingOptions()));
    ExpressionDomainTranslator.ExtractionResult decomposedCubePredicate = ExpressionDomainTranslator.fromPredicate(metadata, session, cubePredicate, types);
    if (!BooleanLiteral.TRUE_LITERAL.equals(decomposedCubePredicate.getRemainingExpression())) {
        // Extract disjunctions from the expression and evaluate separately
        return atLeastMatchesOne(ExpressionUtils.extractDisjuncts(cubePredicate), decomposedQueryPredicate.getTupleDomain(), session, types);
    }
    return decomposedCubePredicate.getTupleDomain().contains(decomposedQueryPredicate.getTupleDomain());
}

Example 5 with CubeFilter

use of io.hetu.core.spi.cube.CubeFilter in project hetu-core by openlookeng.

the class StarTreeAggregationRule method filterPredicateMatches.

private boolean filterPredicateMatches(FilterNode filterNode, CubeMetadata cubeMetadata, Session session, TypeProvider types) {
    CubeFilter cubeFilter = cubeMetadata.getCubeFilter();
    if (cubeFilter == null) {
        // Cube was built for entire table
        return filterNode == null || doesCubeContainQueryPredicateColumns(castToExpression(filterNode.getPredicate()), cubeMetadata);
    }
    if (filterNode == null) {
        // Query statement has no WHERE clause but CUBE was built for subset of original data
        return false;
    }
    SqlParser sqlParser = new SqlParser();
    Expression queryPredicate = castToExpression(filterNode.getPredicate());
    Expression sourceTablePredicate = cubeFilter.getSourceTablePredicate() == null ? null : sqlParser.createExpression(cubeFilter.getSourceTablePredicate(), new ParsingOptions());
    Pair<Expression, Expression> splitQueryPredicate = splitQueryPredicate(queryPredicate, sourceTablePredicate);
    if (!arePredicatesEqual(splitQueryPredicate.getLeft(), sourceTablePredicate, metadata, session, types)) {
        LOGGER.debug("Cube source table predicate %s not matching query predicate %s", sourceTablePredicate, queryPredicate);
        return false;
    }
    // Check if columns in query predicate are all part of the Cube.
    if ((cubeFilter.getCubePredicate() != null && splitQueryPredicate.getRight() == null) || (splitQueryPredicate.getRight() != null && !doesCubeContainQueryPredicateColumns(splitQueryPredicate.getRight(), cubeMetadata))) {
        // Cube does not contain all columns in the remaining predicate
        return false;
    }
    if (cubeFilter.getCubePredicate() == null) {
        // Cube has no additional predicates to compare with. i.e. Cube can be used to optimize the query
        return true;
    }
    Expression cubePredicate = ExpressionUtils.rewriteIdentifiersToSymbolReferences(sqlParser.createExpression(cubeFilter.getCubePredicate(), new ParsingOptions()));
    ExpressionDomainTranslator.ExtractionResult decomposedQueryPredicate = ExpressionDomainTranslator.fromPredicate(metadata, session, splitQueryPredicate.getRight(), types);
    if (!BooleanLiteral.TRUE_LITERAL.equals(decomposedQueryPredicate.getRemainingExpression())) {
        LOGGER.error("StarTree cube cannot support predicate %s", castToExpression(filterNode.getPredicate()));
        return false;
    }
    ExpressionDomainTranslator.ExtractionResult decomposedCubePredicate = ExpressionDomainTranslator.fromPredicate(metadata, session, cubePredicate, types);
    if (!BooleanLiteral.TRUE_LITERAL.equals(decomposedCubePredicate.getRemainingExpression())) {
        // Extract disjuncts from the Expression expression and evaluate separately
        return atLeastMatchesOne(ExpressionUtils.extractDisjuncts(cubePredicate), decomposedQueryPredicate.getTupleDomain(), session, types);
    }
    return decomposedCubePredicate.getTupleDomain().contains(decomposedQueryPredicate.getTupleDomain());
}