Examples with ExpressionType org.voltdb.types.ExpressionType used on opensource projects

Example 16 with ExpressionType

use of org.voltdb.types.ExpressionType in project voltdb by VoltDB.

the class PlannerTestCase method assertExprTopDownTree.

/**
     * Assert that an expression tree contains the expected types of expressions
     * in the order listed, assuming a top-down left-to-right depth-first
     * traversal through left, right, and args children.
     * A null expression type in the list will match any expression
     * node or tree at the corresponding position.
     **/
protected static void assertExprTopDownTree(AbstractExpression start, ExpressionType... exprTypes) {
    assertNotNull(start);
    Stack<AbstractExpression> stack = new Stack<>();
    stack.push(start);
    for (ExpressionType type : exprTypes) {
        // Process each node before its children or later siblings.
        AbstractExpression parent;
        try {
            parent = stack.pop();
        } catch (EmptyStackException ese) {
            fail("No expression was found in the tree to match type " + type);
            // This dead code hushes warnings.
            return;
        }
        List<AbstractExpression> args = parent.getArgs();
        AbstractExpression rightExpr = parent.getRight();
        AbstractExpression leftExpr = parent.getLeft();
        int argCount = (args == null) ? 0 : args.size();
        int childCount = argCount + (rightExpr == null ? 0 : 1) + (leftExpr == null ? 0 : 1);
        if (type == null) {
            // A null type wildcard matches any child TREE or NODE.
            System.out.println("DEBUG: Suggestion -- expect " + parent.getExpressionType() + " with " + childCount + " direct children.");
            continue;
        }
        assertEquals(type, parent.getExpressionType());
        // Iterate from the last child to the first.
        while (argCount > 0) {
            // Push each child to be processed before its parent's
            // or its own later siblings (already pushed).
            stack.push(parent.getArgs().get(--argCount));
        }
        if (rightExpr != null) {
            stack.push(rightExpr);
        }
        if (leftExpr != null) {
            stack.push(leftExpr);
        }
    }
    assertTrue("Extra expression node(s) (" + stack.size() + ") were found in the tree with no expression type to match", stack.isEmpty());
}

Example 17 with ExpressionType

use of org.voltdb.types.ExpressionType in project voltdb by VoltDB.

the class TestPushDownAggregates method checkPushedDown.

private void checkPushedDown(List<AbstractPlanNode> pn, boolean isAggInlined, ExpressionType[] aggTypes, ExpressionType[] pushDownTypes, boolean hasProjectionNode) {
    // Aggregate push down check has to run on two fragments
    assertTrue(pn.size() == 2);
    AbstractPlanNode p = pn.get(0).getChild(0);
    ;
    if (hasProjectionNode) {
        // Complex aggregation or optimized AVG
        assertTrue(p instanceof ProjectionPlanNode);
        p = p.getChild(0);
    }
    assertTrue(p instanceof AggregatePlanNode);
    String fragmentString = p.toJSONString();
    ExpressionType[] topTypes = (pushDownTypes != null) ? pushDownTypes : aggTypes;
    for (ExpressionType type : topTypes) {
        assertTrue(fragmentString.contains("\"AGGREGATE_TYPE\":\"" + type.toString() + "\""));
    }
    // Check the pushed down aggregation
    p = pn.get(1).getChild(0);
    if (pushDownTypes == null) {
        assertTrue(p instanceof AbstractScanPlanNode);
        return;
    }
    if (isAggInlined) {
        // See ENG-6131
        if (p instanceof TableCountPlanNode) {
            return;
        }
        assertTrue(p instanceof AbstractScanPlanNode);
        assertTrue(p.getInlinePlanNode(PlanNodeType.AGGREGATE) != null || p.getInlinePlanNode(PlanNodeType.HASHAGGREGATE) != null);
        if (p.getInlinePlanNode(PlanNodeType.AGGREGATE) != null) {
            p = p.getInlinePlanNode(PlanNodeType.AGGREGATE);
        } else {
            p = p.getInlinePlanNode(PlanNodeType.HASHAGGREGATE);
        }
    } else {
        assertTrue(p instanceof AggregatePlanNode);
    }
    fragmentString = p.toJSONString();
    for (ExpressionType type : aggTypes) {
        assertTrue(fragmentString.contains("\"AGGREGATE_TYPE\":\"" + type.toString() + "\""));
    }
}

Example 18 with ExpressionType

use of org.voltdb.types.ExpressionType in project voltdb by VoltDB.

the class MaterializedViewFixInfo method processMVBasedQueryFix.

/**
     * Check whether the results from a materialized view need to be
     * re-aggregated on the coordinator by the view's GROUP BY columns
     * prior to any of the processing specified by the query.
     * This is normally the case when a mat view's source table is partitioned
     * and the view's GROUP BY does not include the partition key.
     * There is a special edge case where the query already contains the exact
     * reaggregations that the added-cost fix would introduce, so the fix can
     * be skipped as an optimization.
     * Set the m_needed flag to true, only if the reaggregation fix is needed.
     * @return The value of m_needed
     */
public boolean processMVBasedQueryFix(StmtTableScan mvTableScan, Set<SchemaColumn> scanColumns, JoinNode joinTree, List<ParsedColInfo> displayColumns, List<ParsedColInfo> groupByColumns) {
    //@TODO
    if (!(mvTableScan instanceof StmtTargetTableScan)) {
        return false;
    }
    Table table = ((StmtTargetTableScan) mvTableScan).getTargetTable();
    assert (table != null);
    String mvTableName = table.getTypeName();
    Table srcTable = table.getMaterializer();
    if (srcTable == null) {
        return false;
    }
    if (table.getIsreplicated()) {
        return false;
    }
    // Justify whether partition column is in group by column list or not
    if (table.getPartitioncolumn() != null) {
        return false;
    }
    m_mvTableScan = mvTableScan;
    Set<String> mvDDLGroupbyColumnNames = new HashSet<>();
    List<Column> mvColumnArray = CatalogUtil.getSortedCatalogItems(table.getColumns(), "index");
    String mvTableAlias = getMVTableAlias();
    // Get the number of group-by columns.
    int numOfGroupByColumns;
    MaterializedViewInfo mvInfo = srcTable.getViews().get(mvTableName);
    if (mvInfo != null) {
        // single table view
        String complexGroupbyJson = mvInfo.getGroupbyexpressionsjson();
        if (complexGroupbyJson.length() > 0) {
            List<AbstractExpression> mvComplexGroupbyCols = null;
            try {
                mvComplexGroupbyCols = AbstractExpression.fromJSONArrayString(complexGroupbyJson, null);
            } catch (JSONException e) {
                e.printStackTrace();
            }
            numOfGroupByColumns = mvComplexGroupbyCols.size();
        } else {
            numOfGroupByColumns = mvInfo.getGroupbycols().size();
        }
    } else {
        // joined table view
        MaterializedViewHandlerInfo mvHandlerInfo = table.getMvhandlerinfo().get("mvHandlerInfo");
        numOfGroupByColumns = mvHandlerInfo.getGroupbycolumncount();
    }
    if (scanColumns.isEmpty() && numOfGroupByColumns == 0) {
        // This is an edge case that can happen if the view
        // has no group by keys, and we are just
        // doing a count(*) on the output of the view.
        //
        // Having no GB keys or scan columns would cause us to
        // produce plan nodes that have a 0-column output schema.
        // We can't handle this in several places, so add the
        // count(*) column from the view to the scan columns.
        // this is the "count(*)" column.
        Column mvCol = mvColumnArray.get(0);
        TupleValueExpression tve = new TupleValueExpression(mvTableName, mvTableAlias, mvCol, 0);
        tve.setOrigStmtId(mvTableScan.getStatementId());
        String colName = mvCol.getName();
        SchemaColumn scol = new SchemaColumn(mvTableName, mvTableAlias, colName, colName, tve);
        scanColumns.add(scol);
    }
    // Start to do real materialized view processing to fix the duplicates problem.
    // (1) construct new projection columns for scan plan node.
    Set<SchemaColumn> mvDDLGroupbyColumns = new HashSet<>();
    NodeSchema inlineProjSchema = new NodeSchema();
    for (SchemaColumn scol : scanColumns) {
        inlineProjSchema.addColumn(scol);
    }
    for (int i = 0; i < numOfGroupByColumns; i++) {
        Column mvCol = mvColumnArray.get(i);
        String colName = mvCol.getName();
        TupleValueExpression tve = new TupleValueExpression(mvTableName, mvTableAlias, mvCol, i);
        tve.setOrigStmtId(mvTableScan.getStatementId());
        mvDDLGroupbyColumnNames.add(colName);
        SchemaColumn scol = new SchemaColumn(mvTableName, mvTableAlias, colName, colName, tve);
        mvDDLGroupbyColumns.add(scol);
        if (!scanColumns.contains(scol)) {
            scanColumns.add(scol);
            // construct new projection columns for scan plan node.
            inlineProjSchema.addColumn(scol);
        }
    }
    // Record the re-aggregation type for each scan columns.
    Map<String, ExpressionType> mvColumnReAggType = new HashMap<>();
    for (int i = numOfGroupByColumns; i < mvColumnArray.size(); i++) {
        Column mvCol = mvColumnArray.get(i);
        ExpressionType reAggType = ExpressionType.get(mvCol.getAggregatetype());
        if (reAggType == ExpressionType.AGGREGATE_COUNT_STAR || reAggType == ExpressionType.AGGREGATE_COUNT) {
            reAggType = ExpressionType.AGGREGATE_SUM;
        }
        mvColumnReAggType.put(mvCol.getName(), reAggType);
    }
    assert (inlineProjSchema.size() > 0);
    m_scanInlinedProjectionNode = new ProjectionPlanNode(inlineProjSchema);
    // (2) Construct the reAggregation Node.
    // Construct the reAggregation plan node's aggSchema
    m_reAggNode = new HashAggregatePlanNode();
    int outputColumnIndex = 0;
    // inlineProjSchema contains the group by columns, while aggSchema may do not.
    NodeSchema aggSchema = new NodeSchema();
    // Construct reAggregation node's aggregation and group by list.
    for (SchemaColumn scol : inlineProjSchema.getColumns()) {
        if (mvDDLGroupbyColumns.contains(scol)) {
            // Add group by expression.
            m_reAggNode.addGroupByExpression(scol.getExpression());
        } else {
            ExpressionType reAggType = mvColumnReAggType.get(scol.getColumnName());
            assert (reAggType != null);
            AbstractExpression agg_input_expr = scol.getExpression();
            assert (agg_input_expr instanceof TupleValueExpression);
            // Add aggregation information.
            m_reAggNode.addAggregate(reAggType, false, outputColumnIndex, agg_input_expr);
        }
        aggSchema.addColumn(scol);
        outputColumnIndex++;
    }
    assert (aggSchema.size() > 0);
    m_reAggNode.setOutputSchema(aggSchema);
    // Collect all TVEs that need to be do re-aggregation in coordinator.
    List<TupleValueExpression> needReAggTVEs = new ArrayList<>();
    List<AbstractExpression> aggPostExprs = new ArrayList<>();
    for (int i = numOfGroupByColumns; i < mvColumnArray.size(); i++) {
        Column mvCol = mvColumnArray.get(i);
        TupleValueExpression tve = new TupleValueExpression(mvTableName, mvTableAlias, mvCol, -1);
        tve.setOrigStmtId(mvTableScan.getStatementId());
        needReAggTVEs.add(tve);
    }
    collectReAggNodePostExpressions(joinTree, needReAggTVEs, aggPostExprs);
    AbstractExpression aggPostExpr = ExpressionUtil.combinePredicates(aggPostExprs);
    // Add post filters for the reAggregation node.
    m_reAggNode.setPostPredicate(aggPostExpr);
    // ENG-5386
    if (m_edgeCaseQueryNoFixNeeded && edgeCaseQueryNoFixNeeded(mvDDLGroupbyColumnNames, mvColumnReAggType, displayColumns, groupByColumns)) {
        return false;
    }
    m_needed = true;
    return true;
}

Example 19 with ExpressionType

use of org.voltdb.types.ExpressionType in project voltdb by VoltDB.

the class MaterializedViewFixInfo method edgeCaseQueryNoFixNeeded.

/** ENG-5386: do not fix some cases in order to get better performance.
     * There is a special edge case when certain queries are applied to
     * partitioned materialized views that do not contain the partition key in
     * their GROUP BY columns. In this special case, where the query duplicates
     * the reaggregation behavior of the fix -- which must consist of MIN, MAX
     * and/or non-distinct SUM reaggregations -- the added-cost fix code can be
     * skipped as an optimization.
     */
private boolean edgeCaseQueryNoFixNeeded(Set<String> mvDDLGroupbyColumnNames, Map<String, ExpressionType> mvColumnAggType, List<ParsedColInfo> displayColumns, List<ParsedColInfo> groupByColumns) {
    // Condition (1): Group by columns must be part of or all from MV DDL group by TVEs.
    for (ParsedColInfo gcol : groupByColumns) {
        assert (gcol.expression instanceof TupleValueExpression);
        TupleValueExpression tve = (TupleValueExpression) gcol.expression;
        if (tve.getTableName().equals(getMVTableName()) && !mvDDLGroupbyColumnNames.contains(tve.getColumnName())) {
            return false;
        }
    }
    // Condition (2): All the aggregations must qualify.
    for (ParsedColInfo dcol : displayColumns) {
        if (groupByColumns.contains(dcol)) {
            // Skip a group-by column pass-through.
            continue;
        }
        if (dcol.expression instanceof AggregateExpression == false) {
            return false;
        }
        AggregateExpression aggExpr = (AggregateExpression) dcol.expression;
        if (aggExpr.getLeft() instanceof TupleValueExpression == false) {
            return false;
        }
        ExpressionType type = aggExpr.getExpressionType();
        // can tolerate a skipped reaggregation.
        if ((type != ExpressionType.AGGREGATE_SUM || aggExpr.isDistinct()) && type != ExpressionType.AGGREGATE_MIN && type != ExpressionType.AGGREGATE_MAX) {
            return false;
        }
        TupleValueExpression tve = (TupleValueExpression) aggExpr.getLeft();
        if (tve.getTableName().equals(getMVTableName())) {
            String columnName = tve.getColumnName();
            // SUMming a SUM, MINning a MIN, or MAXxing a MAX.
            if (mvColumnAggType.get(columnName) != type) {
                return false;
            }
        } else {
            // The duplication would corrupt a SUM.
            if (type == ExpressionType.AGGREGATE_SUM) {
                return false;
            }
        }
    }
    // Edge case query can be optimized with correct answer without MV reAggregation fix.
    return true;
}

Example 20 with ExpressionType

use of org.voltdb.types.ExpressionType in project voltdb by VoltDB.

the class ParsedUnionStmt method breakUpSetOpSubquery.

/**
     * Break up UNION/INTERSECT (ALL) set ops into individual selects that are part
     * of the IN/EXISTS subquery into multiple expressions for each set op child
     * combined by the conjunction AND/OR expression.
     * col IN ( queryA UNION queryB ) - > col IN (queryA) OR col IN (queryB)
     * col IN ( queryA INTERSECTS queryB ) - > col IN (queryA) AND col IN (queryB)
     * The EXCEPT set op is LEFT as is
     * Also the ALL qualifier is dropped because IN/EXISTS expressions only
     * need just one tuple in the results set
     *
     * @param subqueryExpr - IN/EXISTS expression with a possible SET OP subquery
     * @return simplified expression
     */
protected static AbstractExpression breakUpSetOpSubquery(AbstractExpression expr) {
    assert (expr != null);
    SelectSubqueryExpression subqueryExpr = null;
    if (expr.getExpressionType() == ExpressionType.COMPARE_EQUAL && expr.getRight() instanceof SelectSubqueryExpression) {
        subqueryExpr = (SelectSubqueryExpression) expr.getRight();
    } else if (expr.getExpressionType() == ExpressionType.OPERATOR_EXISTS && expr.getLeft() instanceof SelectSubqueryExpression) {
        subqueryExpr = (SelectSubqueryExpression) expr.getLeft();
    }
    if (subqueryExpr == null) {
        return expr;
    }
    AbstractParsedStmt subquery = subqueryExpr.getSubqueryStmt();
    if (!(subquery instanceof ParsedUnionStmt)) {
        return expr;
    }
    ParsedUnionStmt setOpStmt = (ParsedUnionStmt) subquery;
    if (UnionType.EXCEPT == setOpStmt.m_unionType || UnionType.EXCEPT_ALL == setOpStmt.m_unionType) {
        setOpStmt.m_unionType = UnionType.EXCEPT;
        return expr;
    }
    if (UnionType.UNION_ALL == setOpStmt.m_unionType) {
        setOpStmt.m_unionType = UnionType.UNION;
    } else if (UnionType.INTERSECT_ALL == setOpStmt.m_unionType) {
        setOpStmt.m_unionType = UnionType.INTERSECT;
    }
    ExpressionType conjuctionType = (setOpStmt.m_unionType == UnionType.UNION) ? ExpressionType.CONJUNCTION_OR : ExpressionType.CONJUNCTION_AND;
    AbstractExpression retval = null;
    AbstractParsedStmt parentStmt = subquery.m_parentStmt;
    // It's a subquery which means it must have a parent
    assert (parentStmt != null);
    for (AbstractParsedStmt child : setOpStmt.m_children) {
        // add table to the query cache
        String withoutAlias = null;
        StmtSubqueryScan tableCache = parentStmt.addSubqueryToStmtCache(child, withoutAlias);
        AbstractExpression childSubqueryExpr = new SelectSubqueryExpression(subqueryExpr.getExpressionType(), tableCache);
        AbstractExpression newExpr = null;
        try {
            newExpr = expr.getExpressionType().getExpressionClass().newInstance();
        } catch (Exception e) {
            e.printStackTrace();
            throw new RuntimeException(e.getMessage(), e);
        }
        newExpr.setExpressionType(expr.getExpressionType());
        if (ExpressionType.COMPARE_EQUAL == expr.getExpressionType()) {
            newExpr.setLeft(expr.getLeft().clone());
            newExpr.setRight(childSubqueryExpr);
            assert (newExpr instanceof ComparisonExpression);
            ((ComparisonExpression) newExpr).setQuantifier(((ComparisonExpression) expr).getQuantifier());
        } else {
            newExpr.setLeft(childSubqueryExpr);
        }
        // Recurse
        newExpr = ParsedUnionStmt.breakUpSetOpSubquery(newExpr);
        if (retval == null) {
            retval = newExpr;
        } else {
            retval = new ConjunctionExpression(conjuctionType, retval, newExpr);
        }
    }
    return retval;
}