Examples with ParsingOptions io.prestosql.sql.parser.ParsingOptions used on opensource projects

Example 11 with ParsingOptions

use of io.prestosql.sql.parser.ParsingOptions in project hetu-core by openlookeng.

the class CubeOptimizer method filterPredicateMatches.

private boolean filterPredicateMatches(Expression queryPredicate, CubeMetadata cubeMetadata, Session session, TypeProvider types) {
    CubeFilter cubeFilter = cubeMetadata.getCubeFilter();
    if (cubeFilter == null) {
        // Cube was built for entire table
        return queryPredicate == null || doesCubeContainQueryPredicateColumns(queryPredicate, cubeMetadata);
    }
    if (queryPredicate == null) {
        // Query statement has no WHERE clause but CUBE was built for subset of original data
        return false;
    }
    SqlParser sqlParser = new SqlParser();
    Expression cubeSourceTablePredicate = cubeFilter.getSourceTablePredicate() == null ? null : sqlParser.createExpression(cubeFilter.getSourceTablePredicate(), new ParsingOptions());
    Pair<Expression, Expression> queryPredicateSplit = splitQueryPredicate(queryPredicate, cubeSourceTablePredicate);
    if (!arePredicatesEqual(queryPredicateSplit.getLeft(), cubeSourceTablePredicate, metadata, session, types)) {
        log.debug("Cube source table predicate %s not matching query predicate %s", cubeSourceTablePredicate, queryPredicate);
        return false;
    }
    // Check if columns in query predicate are all part of the Cube.
    if ((cubeFilter.getCubePredicate() != null && queryPredicateSplit.getRight() == null) || (queryPredicateSplit.getRight() != null && !doesCubeContainQueryPredicateColumns(queryPredicateSplit.getRight(), cubeMetadata))) {
        // Cube does not contain all columns in the remaining predicate
        return false;
    }
    if (cubeFilter.getCubePredicate() == null) {
        // Cube has no additional predicates to compare with. i.e. Cube can be used to optimize the query
        return true;
    }
    ExpressionDomainTranslator.ExtractionResult decomposedQueryPredicate = ExpressionDomainTranslator.fromPredicate(metadata, session, queryPredicateSplit.getRight(), types);
    if (!BooleanLiteral.TRUE_LITERAL.equals(decomposedQueryPredicate.getRemainingExpression())) {
        log.error("StarTree cube cannot support predicate %s", castToExpression(filterNode.getPredicate()));
        return false;
    }
    Expression cubePredicate = ExpressionUtils.rewriteIdentifiersToSymbolReferences(sqlParser.createExpression(cubeFilter.getCubePredicate(), new ParsingOptions()));
    ExpressionDomainTranslator.ExtractionResult decomposedCubePredicate = ExpressionDomainTranslator.fromPredicate(metadata, session, cubePredicate, types);
    if (!BooleanLiteral.TRUE_LITERAL.equals(decomposedCubePredicate.getRemainingExpression())) {
        // Extract disjunctions from the expression and evaluate separately
        return atLeastMatchesOne(ExpressionUtils.extractDisjuncts(cubePredicate), decomposedQueryPredicate.getTupleDomain(), session, types);
    }
    return decomposedCubePredicate.getTupleDomain().contains(decomposedQueryPredicate.getTupleDomain());
}

Example 12 with ParsingOptions

use of io.prestosql.sql.parser.ParsingOptions in project hetu-core by openlookeng.

the class StarTreeAggregationRule method rewriteByRemovingSourceFilter.

private FilterNode rewriteByRemovingSourceFilter(PlanNode filterNode, CubeMetadata matchedCubeMetadata) {
    FilterNode rewritten = (FilterNode) filterNode;
    if (filterNode != null && matchedCubeMetadata.getCubeFilter() != null && matchedCubeMetadata.getCubeFilter().getSourceTablePredicate() != null) {
        // rewrite the expression by removing source filter predicate as cube would not have those columns necessarily
        Expression predicate = castToExpression(((FilterNode) filterNode).getPredicate());
        SqlParser sqlParser = new SqlParser();
        Set<Identifier> sourceFilterPredicateColumns = ExpressionUtils.getIdentifiers(sqlParser.createExpression(matchedCubeMetadata.getCubeFilter().getSourceTablePredicate(), new ParsingOptions()));
        predicate = ExpressionUtils.filterConjuncts(predicate, conjunct -> !sourceFilterPredicateColumns.containsAll(SymbolsExtractor.extractUnique(conjunct).stream().map(Symbol::getName).map(Identifier::new).collect(Collectors.toList())));
        rewritten = new FilterNode(filterNode.getId(), ((FilterNode) filterNode).getSource(), castToRowExpression(predicate));
    }
    return rewritten;
}

Example 13 with ParsingOptions

use of io.prestosql.sql.parser.ParsingOptions in project hetu-core by openlookeng.

the class StarTreeAggregationRule method filterPredicateMatches.

private boolean filterPredicateMatches(FilterNode filterNode, CubeMetadata cubeMetadata, Session session, TypeProvider types) {
    CubeFilter cubeFilter = cubeMetadata.getCubeFilter();
    if (cubeFilter == null) {
        // Cube was built for entire table
        return filterNode == null || doesCubeContainQueryPredicateColumns(castToExpression(filterNode.getPredicate()), cubeMetadata);
    }
    if (filterNode == null) {
        // Query statement has no WHERE clause but CUBE was built for subset of original data
        return false;
    }
    SqlParser sqlParser = new SqlParser();
    Expression queryPredicate = castToExpression(filterNode.getPredicate());
    Expression sourceTablePredicate = cubeFilter.getSourceTablePredicate() == null ? null : sqlParser.createExpression(cubeFilter.getSourceTablePredicate(), new ParsingOptions());
    Pair<Expression, Expression> splitQueryPredicate = splitQueryPredicate(queryPredicate, sourceTablePredicate);
    if (!arePredicatesEqual(splitQueryPredicate.getLeft(), sourceTablePredicate, metadata, session, types)) {
        LOGGER.debug("Cube source table predicate %s not matching query predicate %s", sourceTablePredicate, queryPredicate);
        return false;
    }
    // Check if columns in query predicate are all part of the Cube.
    if ((cubeFilter.getCubePredicate() != null && splitQueryPredicate.getRight() == null) || (splitQueryPredicate.getRight() != null && !doesCubeContainQueryPredicateColumns(splitQueryPredicate.getRight(), cubeMetadata))) {
        // Cube does not contain all columns in the remaining predicate
        return false;
    }
    if (cubeFilter.getCubePredicate() == null) {
        // Cube has no additional predicates to compare with. i.e. Cube can be used to optimize the query
        return true;
    }
    Expression cubePredicate = ExpressionUtils.rewriteIdentifiersToSymbolReferences(sqlParser.createExpression(cubeFilter.getCubePredicate(), new ParsingOptions()));
    ExpressionDomainTranslator.ExtractionResult decomposedQueryPredicate = ExpressionDomainTranslator.fromPredicate(metadata, session, splitQueryPredicate.getRight(), types);
    if (!BooleanLiteral.TRUE_LITERAL.equals(decomposedQueryPredicate.getRemainingExpression())) {
        LOGGER.error("StarTree cube cannot support predicate %s", castToExpression(filterNode.getPredicate()));
        return false;
    }
    ExpressionDomainTranslator.ExtractionResult decomposedCubePredicate = ExpressionDomainTranslator.fromPredicate(metadata, session, cubePredicate, types);
    if (!BooleanLiteral.TRUE_LITERAL.equals(decomposedCubePredicate.getRemainingExpression())) {
        // Extract disjuncts from the Expression expression and evaluate separately
        return atLeastMatchesOne(ExpressionUtils.extractDisjuncts(cubePredicate), decomposedQueryPredicate.getTupleDomain(), session, types);
    }
    return decomposedCubePredicate.getTupleDomain().contains(decomposedQueryPredicate.getTupleDomain());
}

Example 14 with ParsingOptions

use of io.prestosql.sql.parser.ParsingOptions in project hetu-core by openlookeng.

the class TestExpressionEquivalence method assertEquivalent.

private static void assertEquivalent(@Language("SQL") String left, @Language("SQL") String right) {
    ParsingOptions parsingOptions = new ParsingOptions(AS_DOUBLE);
    Expression leftExpression = rewriteIdentifiersToSymbolReferences(SQL_PARSER.createExpression(left, parsingOptions));
    Expression rightExpression = rewriteIdentifiersToSymbolReferences(SQL_PARSER.createExpression(right, parsingOptions));
    Set<Symbol> symbols = extractUnique(ImmutableList.of(leftExpression, rightExpression));
    TypeProvider types = TypeProvider.copyOf(symbols.stream().collect(toMap(identity(), TestExpressionEquivalence::generateType)));
    assertTrue(EQUIVALENCE.areExpressionsEquivalent(TEST_SESSION, leftExpression, rightExpression, types), format("Expected (%s) and (%s) to be equivalent", left, right));
    assertTrue(EQUIVALENCE.areExpressionsEquivalent(TEST_SESSION, rightExpression, leftExpression, types), format("Expected (%s) and (%s) to be equivalent", right, left));
}

Example 15 with ParsingOptions

use of io.prestosql.sql.parser.ParsingOptions in project hetu-core by openlookeng.

the class CubeConsole method processComparisonExpression.

/**
 * Process the Create Cube Query with Comparison Expression in where clause.
 *
 * @param createCube createCube
 * @param queryRunner queryRunner
 * @param outputFormat outputFormat
 * @param schemaChanged schemaChanged
 * @param usePager usePager
 * @param showProgress showProgress
 * @param terminal terminal
 * @param out out
 * @param errorChannel errorChannel
 * @param parser parser
 * @return boolean
 */
private boolean processComparisonExpression(CreateCube createCube, QueryRunner queryRunner, ClientOptions.OutputFormat outputFormat, Runnable schemaChanged, boolean usePager, boolean showProgress, Terminal terminal, PrintStream out, PrintStream errorChannel, SqlParser parser) {
    String whereClause = createCube.getWhere().get().toString();
    QualifiedName sourceTableName = createCube.getSourceTableName();
    QualifiedName cubeName = createCube.getCubeName();
    ComparisonExpression comparisonExpression = (ComparisonExpression) (createCube.getWhere().get());
    ComparisonExpression.Operator operator = comparisonExpression.getOperator();
    Expression left = comparisonExpression.getLeft();
    Expression right = comparisonExpression.getRight();
    boolean notEqualOperator = false;
    if (operator.getValue().equalsIgnoreCase(NOT_EQUAL_OPERATOR)) {
        notEqualOperator = true;
    }
    if (!(left instanceof SymbolReference) && right instanceof SymbolReference) {
        comparisonExpression = new ComparisonExpression(operator.flip(), right, left);
    }
    if (left instanceof Literal && !(right instanceof Literal)) {
        comparisonExpression = new ComparisonExpression(operator.flip(), right, left);
    }
    Expression columnName = comparisonExpression.getLeft();
    // Run Query
    String rowCountsDistinctValuesQuery = String.format(SELECT_COLUMN_ROW_COUNT_FROM_STRING, columnName, sourceTableName.toString(), whereClause, columnName, columnName);
    if (!processCubeInitialQuery(queryRunner, rowCountsDistinctValuesQuery, outputFormat, schemaChanged, usePager, showProgress, terminal, out, errorChannel)) {
        return false;
    }
    List<List<?>> bufferIterationItems = getListRowBufferIterationItems();
    if (bufferIterationItems != null && bufferIterationItems.size() != EMPTY_ROW_BUFFER_ITERATION_ITEMS) {
        // this loop process the multiple insert query statements
        int end = bufferIterationItems.size() - 1;
        for (int i = 0; i <= end; i++) {
            List<?> rowBufferItems = bufferIterationItems.get(i);
            Expression finalPredicate;
            Expression userBoundaryPredicate = null;
            String queryInsert;
            String minItem = rowBufferItems.get(INDEX_AT_MIN_POSITION).toString();
            String maxItem = rowBufferItems.get(INDEX_AT_MAX_POSITION).toString();
            switch(cubeColumnDataType) {
                case DATATYPE_DOUBLE:
                    {
                        finalPredicate = new BetweenPredicate(columnName, parser.createExpression(DATATYPE_DOUBLE + " " + QUOTE_STRING + minItem + QUOTE_STRING, new ParsingOptions()), parser.createExpression(DATATYPE_DOUBLE + " " + QUOTE_STRING + maxItem + QUOTE_STRING, new ParsingOptions()));
                        userBoundaryPredicate = new ComparisonExpression(operator, left, right);
                        break;
                    }
                case DATATYPE_REAL:
                    {
                        finalPredicate = new BetweenPredicate(columnName, parser.createExpression(DATATYPE_REAL_QUOTE + minItem + QUOTE_STRING, new ParsingOptions()), parser.createExpression(DATATYPE_REAL_QUOTE + maxItem + QUOTE_STRING, new ParsingOptions()));
                        userBoundaryPredicate = new ComparisonExpression(operator, left, right);
                        break;
                    }
                case DATATYPE_DECIMAL:
                    {
                        finalPredicate = new BetweenPredicate(columnName, parser.createExpression(DATATYPE_DECIMAL + " " + QUOTE_STRING + minItem + QUOTE_STRING, new ParsingOptions()), parser.createExpression(DATATYPE_DECIMAL + " " + QUOTE_STRING + maxItem + QUOTE_STRING, new ParsingOptions()));
                        userBoundaryPredicate = new ComparisonExpression(operator, left, right);
                        break;
                    }
                case DATATYPE_DATE:
                    {
                        finalPredicate = new BetweenPredicate(left, parser.createExpression(DATATYPE_DATE_QUOTE + minItem + QUOTE_STRING, new ParsingOptions()), parser.createExpression(DATATYPE_DATE_QUOTE + maxItem + QUOTE_STRING, new ParsingOptions()));
                        userBoundaryPredicate = new ComparisonExpression(operator, left, right);
                        break;
                    }
                case DATATYPE_TIMESTAMP:
                    {
                        finalPredicate = new BetweenPredicate(left, parser.createExpression(DATATYPE_TIMESTAMP_QUOTE + minItem + QUOTE_STRING, new ParsingOptions()), parser.createExpression(DATATYPE_TIMESTAMP_QUOTE + maxItem + QUOTE_STRING, new ParsingOptions()));
                        userBoundaryPredicate = new ComparisonExpression(operator, left, right);
                        break;
                    }
                case DATATYPE_TINYINT:
                    {
                        finalPredicate = new BetweenPredicate(left, parser.createExpression(DATATYPE_TINYINT_QUOTE + minItem + QUOTE_STRING, new ParsingOptions()), parser.createExpression(DATATYPE_TINYINT_QUOTE + maxItem + QUOTE_STRING, new ParsingOptions()));
                        userBoundaryPredicate = new ComparisonExpression(operator, left, right);
                        break;
                    }
                case DATATYPE_BIGINT:
                    {
                        finalPredicate = new BetweenPredicate(left, parser.createExpression(DATATYPE_BIGINT_QUOTE + minItem + QUOTE_STRING, new ParsingOptions()), parser.createExpression(DATATYPE_BIGINT_QUOTE + maxItem + QUOTE_STRING, new ParsingOptions()));
                        userBoundaryPredicate = new ComparisonExpression(operator, left, right);
                        break;
                    }
                case DATATYPE_SMALLINT:
                    {
                        finalPredicate = new BetweenPredicate(left, parser.createExpression(DATATYPE_SMALLINT_QUOTE + minItem + QUOTE_STRING, new ParsingOptions()), parser.createExpression(DATATYPE_SMALLINT_QUOTE + maxItem + QUOTE_STRING, new ParsingOptions()));
                        userBoundaryPredicate = new ComparisonExpression(operator, left, right);
                        break;
                    }
                case DATATYPE_VARCHAR:
                    {
                        finalPredicate = new BetweenPredicate(left, parser.createExpression(QUOTE_STRING + minItem + QUOTE_STRING, new ParsingOptions()), parser.createExpression(QUOTE_STRING + maxItem + QUOTE_STRING, new ParsingOptions()));
                        userBoundaryPredicate = new ComparisonExpression(operator, left, right);
                        break;
                    }
                default:
                    {
                        finalPredicate = new BetweenPredicate(left, parser.createExpression(minItem, new ParsingOptions()), parser.createExpression(maxItem, new ParsingOptions()));
                        userBoundaryPredicate = new ComparisonExpression(operator, left, right);
                        break;
                    }
            }
            if (notEqualOperator) {
                finalPredicate = new ComparisonExpression(ComparisonExpression.Operator.NOT_EQUAL, left, right);
                queryInsert = String.format(INSERT_INTO_CUBE_STRING, cubeName, finalPredicate);
            } else if (i == end && userBoundaryPredicate != null) {
                queryInsert = String.format(INSERT_INTO_CUBE_STRING, cubeName, userBoundaryPredicate);
            } else {
                queryInsert = String.format(INSERT_INTO_CUBE_STRING, cubeName, finalPredicate);
            }
            if (!console.runQuery(queryRunner, queryInsert, outputFormat, schemaChanged, usePager, showProgress, terminal, out, errorChannel)) {
                return false;
            }
        }
    } else {
        // if the range is within the processing size limit then we run a single insert query only
        String queryInsert = String.format(INSERT_INTO_CUBE_STRING, cubeName, whereClause);
        if (!console.runQuery(queryRunner, queryInsert, outputFormat, schemaChanged, usePager, showProgress, terminal, out, errorChannel)) {
            return false;
        }
    }
    return true;
}