Examples with AbstractExpression org.voltdb.expressions.AbstractExpression used on opensource projects

Example 91 with AbstractExpression

use of org.voltdb.expressions.AbstractExpression in project voltdb by VoltDB.

the class MaterializedViewProcessor method setGroupedTablePartitionColumn.

private void setGroupedTablePartitionColumn(MaterializedViewInfo mvi, Column partitionColumn) throws VoltCompilerException {
    // A view of a replicated table is replicated.
    // A view of a partitioned table is partitioned -- regardless of whether it has a partition key
    // -- it certainly isn't replicated!
    // If the partitioning column is grouped, its counterpart is the partitioning column of the view table.
    // Otherwise, the view table just doesn't have a partitioning column
    // -- it is seemingly randomly distributed,
    // and its grouped columns are only locally unique but not globally unique.
    Table destTable = mvi.getDest();
    // Get the grouped columns in "index" order.
    // This order corresponds to the iteration order of the MaterializedViewInfo's group by columns.
    List<Column> destColumnArray = CatalogUtil.getSortedCatalogItems(destTable.getColumns(), "index");
    // Note getTypeName gets the column name -- go figure.
    String partitionColName = partitionColumn.getTypeName();
    if (mvi.getGroupbycols().size() > 0) {
        int index = 0;
        for (ColumnRef cref : CatalogUtil.getSortedCatalogItems(mvi.getGroupbycols(), "index")) {
            Column srcCol = cref.getColumn();
            if (srcCol.getName().equals(partitionColName)) {
                Column destCol = destColumnArray.get(index);
                destTable.setPartitioncolumn(destCol);
                return;
            }
            ++index;
        }
    } else {
        String complexGroupbyJson = mvi.getGroupbyexpressionsjson();
        if (complexGroupbyJson.length() > 0) {
            int partitionColIndex = partitionColumn.getIndex();
            List<AbstractExpression> mvComplexGroupbyCols = null;
            try {
                mvComplexGroupbyCols = AbstractExpression.fromJSONArrayString(complexGroupbyJson, null);
            } catch (JSONException e) {
                e.printStackTrace();
            }
            int index = 0;
            for (AbstractExpression expr : mvComplexGroupbyCols) {
                if (expr instanceof TupleValueExpression) {
                    TupleValueExpression tve = (TupleValueExpression) expr;
                    if (tve.getColumnIndex() == partitionColIndex) {
                        Column destCol = destColumnArray.get(index);
                        destTable.setPartitioncolumn(destCol);
                        return;
                    }
                }
                ++index;
            }
        }
    }
}

Example 92 with AbstractExpression

use of org.voltdb.expressions.AbstractExpression in project voltdb by VoltDB.

the class MaterializedViewFixInfo method processMVBasedQueryFix.

/**
     * Check whether the results from a materialized view need to be
     * re-aggregated on the coordinator by the view's GROUP BY columns
     * prior to any of the processing specified by the query.
     * This is normally the case when a mat view's source table is partitioned
     * and the view's GROUP BY does not include the partition key.
     * There is a special edge case where the query already contains the exact
     * reaggregations that the added-cost fix would introduce, so the fix can
     * be skipped as an optimization.
     * Set the m_needed flag to true, only if the reaggregation fix is needed.
     * @return The value of m_needed
     */
public boolean processMVBasedQueryFix(StmtTableScan mvTableScan, Set<SchemaColumn> scanColumns, JoinNode joinTree, List<ParsedColInfo> displayColumns, List<ParsedColInfo> groupByColumns) {
    //@TODO
    if (!(mvTableScan instanceof StmtTargetTableScan)) {
        return false;
    }
    Table table = ((StmtTargetTableScan) mvTableScan).getTargetTable();
    assert (table != null);
    String mvTableName = table.getTypeName();
    Table srcTable = table.getMaterializer();
    if (srcTable == null) {
        return false;
    }
    if (table.getIsreplicated()) {
        return false;
    }
    // Justify whether partition column is in group by column list or not
    if (table.getPartitioncolumn() != null) {
        return false;
    }
    m_mvTableScan = mvTableScan;
    Set<String> mvDDLGroupbyColumnNames = new HashSet<>();
    List<Column> mvColumnArray = CatalogUtil.getSortedCatalogItems(table.getColumns(), "index");
    String mvTableAlias = getMVTableAlias();
    // Get the number of group-by columns.
    int numOfGroupByColumns;
    MaterializedViewInfo mvInfo = srcTable.getViews().get(mvTableName);
    if (mvInfo != null) {
        // single table view
        String complexGroupbyJson = mvInfo.getGroupbyexpressionsjson();
        if (complexGroupbyJson.length() > 0) {
            List<AbstractExpression> mvComplexGroupbyCols = null;
            try {
                mvComplexGroupbyCols = AbstractExpression.fromJSONArrayString(complexGroupbyJson, null);
            } catch (JSONException e) {
                e.printStackTrace();
            }
            numOfGroupByColumns = mvComplexGroupbyCols.size();
        } else {
            numOfGroupByColumns = mvInfo.getGroupbycols().size();
        }
    } else {
        // joined table view
        MaterializedViewHandlerInfo mvHandlerInfo = table.getMvhandlerinfo().get("mvHandlerInfo");
        numOfGroupByColumns = mvHandlerInfo.getGroupbycolumncount();
    }
    if (scanColumns.isEmpty() && numOfGroupByColumns == 0) {
        // This is an edge case that can happen if the view
        // has no group by keys, and we are just
        // doing a count(*) on the output of the view.
        //
        // Having no GB keys or scan columns would cause us to
        // produce plan nodes that have a 0-column output schema.
        // We can't handle this in several places, so add the
        // count(*) column from the view to the scan columns.
        // this is the "count(*)" column.
        Column mvCol = mvColumnArray.get(0);
        TupleValueExpression tve = new TupleValueExpression(mvTableName, mvTableAlias, mvCol, 0);
        tve.setOrigStmtId(mvTableScan.getStatementId());
        String colName = mvCol.getName();
        SchemaColumn scol = new SchemaColumn(mvTableName, mvTableAlias, colName, colName, tve);
        scanColumns.add(scol);
    }
    // Start to do real materialized view processing to fix the duplicates problem.
    // (1) construct new projection columns for scan plan node.
    Set<SchemaColumn> mvDDLGroupbyColumns = new HashSet<>();
    NodeSchema inlineProjSchema = new NodeSchema();
    for (SchemaColumn scol : scanColumns) {
        inlineProjSchema.addColumn(scol);
    }
    for (int i = 0; i < numOfGroupByColumns; i++) {
        Column mvCol = mvColumnArray.get(i);
        String colName = mvCol.getName();
        TupleValueExpression tve = new TupleValueExpression(mvTableName, mvTableAlias, mvCol, i);
        tve.setOrigStmtId(mvTableScan.getStatementId());
        mvDDLGroupbyColumnNames.add(colName);
        SchemaColumn scol = new SchemaColumn(mvTableName, mvTableAlias, colName, colName, tve);
        mvDDLGroupbyColumns.add(scol);
        if (!scanColumns.contains(scol)) {
            scanColumns.add(scol);
            // construct new projection columns for scan plan node.
            inlineProjSchema.addColumn(scol);
        }
    }
    // Record the re-aggregation type for each scan columns.
    Map<String, ExpressionType> mvColumnReAggType = new HashMap<>();
    for (int i = numOfGroupByColumns; i < mvColumnArray.size(); i++) {
        Column mvCol = mvColumnArray.get(i);
        ExpressionType reAggType = ExpressionType.get(mvCol.getAggregatetype());
        if (reAggType == ExpressionType.AGGREGATE_COUNT_STAR || reAggType == ExpressionType.AGGREGATE_COUNT) {
            reAggType = ExpressionType.AGGREGATE_SUM;
        }
        mvColumnReAggType.put(mvCol.getName(), reAggType);
    }
    assert (inlineProjSchema.size() > 0);
    m_scanInlinedProjectionNode = new ProjectionPlanNode(inlineProjSchema);
    // (2) Construct the reAggregation Node.
    // Construct the reAggregation plan node's aggSchema
    m_reAggNode = new HashAggregatePlanNode();
    int outputColumnIndex = 0;
    // inlineProjSchema contains the group by columns, while aggSchema may do not.
    NodeSchema aggSchema = new NodeSchema();
    // Construct reAggregation node's aggregation and group by list.
    for (SchemaColumn scol : inlineProjSchema.getColumns()) {
        if (mvDDLGroupbyColumns.contains(scol)) {
            // Add group by expression.
            m_reAggNode.addGroupByExpression(scol.getExpression());
        } else {
            ExpressionType reAggType = mvColumnReAggType.get(scol.getColumnName());
            assert (reAggType != null);
            AbstractExpression agg_input_expr = scol.getExpression();
            assert (agg_input_expr instanceof TupleValueExpression);
            // Add aggregation information.
            m_reAggNode.addAggregate(reAggType, false, outputColumnIndex, agg_input_expr);
        }
        aggSchema.addColumn(scol);
        outputColumnIndex++;
    }
    assert (aggSchema.size() > 0);
    m_reAggNode.setOutputSchema(aggSchema);
    // Collect all TVEs that need to be do re-aggregation in coordinator.
    List<TupleValueExpression> needReAggTVEs = new ArrayList<>();
    List<AbstractExpression> aggPostExprs = new ArrayList<>();
    for (int i = numOfGroupByColumns; i < mvColumnArray.size(); i++) {
        Column mvCol = mvColumnArray.get(i);
        TupleValueExpression tve = new TupleValueExpression(mvTableName, mvTableAlias, mvCol, -1);
        tve.setOrigStmtId(mvTableScan.getStatementId());
        needReAggTVEs.add(tve);
    }
    collectReAggNodePostExpressions(joinTree, needReAggTVEs, aggPostExprs);
    AbstractExpression aggPostExpr = ExpressionUtil.combinePredicates(aggPostExprs);
    // Add post filters for the reAggregation node.
    m_reAggNode.setPostPredicate(aggPostExpr);
    // ENG-5386
    if (m_edgeCaseQueryNoFixNeeded && edgeCaseQueryNoFixNeeded(mvDDLGroupbyColumnNames, mvColumnReAggType, displayColumns, groupByColumns)) {
        return false;
    }
    m_needed = true;
    return true;
}

Example 93 with AbstractExpression

use of org.voltdb.expressions.AbstractExpression in project voltdb by VoltDB.

the class MaterializedViewFixInfo method processFilters.

private AbstractExpression processFilters(AbstractExpression filters, List<TupleValueExpression> needReAggTVEs, List<AbstractExpression> aggPostExprs) {
    if (filters == null) {
        return null;
    }
    // Collect all TVEs that need re-aggregation in the coordinator.
    List<AbstractExpression> remaningExprs = new ArrayList<>();
    // Check where clause.
    List<AbstractExpression> exprs = ExpressionUtil.uncombinePredicate(filters);
    for (AbstractExpression expr : exprs) {
        List<TupleValueExpression> tves = expr.findAllTupleValueSubexpressions();
        boolean canPushdown = true;
        for (TupleValueExpression needReAggTVE : needReAggTVEs) {
            if (tves.contains(needReAggTVE)) {
                m_edgeCaseQueryNoFixNeeded = false;
                if (fromMVTableOnly(tves)) {
                    canPushdown = false;
                }
                break;
            }
        }
        if (canPushdown) {
            remaningExprs.add(expr);
        } else {
            aggPostExprs.add(expr);
        }
    }
    AbstractExpression remaningFilters = ExpressionUtil.combinePredicates(remaningExprs);
    // Update new filters for the scanNode.
    return remaningFilters;
}

Example 94 with AbstractExpression

use of org.voltdb.expressions.AbstractExpression in project voltdb by VoltDB.

the class ParsedUnionStmt method breakUpSetOpSubquery.

/**
     * Break up UNION/INTERSECT (ALL) set ops into individual selects that are part
     * of the IN/EXISTS subquery into multiple expressions for each set op child
     * combined by the conjunction AND/OR expression.
     * col IN ( queryA UNION queryB ) - > col IN (queryA) OR col IN (queryB)
     * col IN ( queryA INTERSECTS queryB ) - > col IN (queryA) AND col IN (queryB)
     * The EXCEPT set op is LEFT as is
     * Also the ALL qualifier is dropped because IN/EXISTS expressions only
     * need just one tuple in the results set
     *
     * @param subqueryExpr - IN/EXISTS expression with a possible SET OP subquery
     * @return simplified expression
     */
protected static AbstractExpression breakUpSetOpSubquery(AbstractExpression expr) {
    assert (expr != null);
    SelectSubqueryExpression subqueryExpr = null;
    if (expr.getExpressionType() == ExpressionType.COMPARE_EQUAL && expr.getRight() instanceof SelectSubqueryExpression) {
        subqueryExpr = (SelectSubqueryExpression) expr.getRight();
    } else if (expr.getExpressionType() == ExpressionType.OPERATOR_EXISTS && expr.getLeft() instanceof SelectSubqueryExpression) {
        subqueryExpr = (SelectSubqueryExpression) expr.getLeft();
    }
    if (subqueryExpr == null) {
        return expr;
    }
    AbstractParsedStmt subquery = subqueryExpr.getSubqueryStmt();
    if (!(subquery instanceof ParsedUnionStmt)) {
        return expr;
    }
    ParsedUnionStmt setOpStmt = (ParsedUnionStmt) subquery;
    if (UnionType.EXCEPT == setOpStmt.m_unionType || UnionType.EXCEPT_ALL == setOpStmt.m_unionType) {
        setOpStmt.m_unionType = UnionType.EXCEPT;
        return expr;
    }
    if (UnionType.UNION_ALL == setOpStmt.m_unionType) {
        setOpStmt.m_unionType = UnionType.UNION;
    } else if (UnionType.INTERSECT_ALL == setOpStmt.m_unionType) {
        setOpStmt.m_unionType = UnionType.INTERSECT;
    }
    ExpressionType conjuctionType = (setOpStmt.m_unionType == UnionType.UNION) ? ExpressionType.CONJUNCTION_OR : ExpressionType.CONJUNCTION_AND;
    AbstractExpression retval = null;
    AbstractParsedStmt parentStmt = subquery.m_parentStmt;
    // It's a subquery which means it must have a parent
    assert (parentStmt != null);
    for (AbstractParsedStmt child : setOpStmt.m_children) {
        // add table to the query cache
        String withoutAlias = null;
        StmtSubqueryScan tableCache = parentStmt.addSubqueryToStmtCache(child, withoutAlias);
        AbstractExpression childSubqueryExpr = new SelectSubqueryExpression(subqueryExpr.getExpressionType(), tableCache);
        AbstractExpression newExpr = null;
        try {
            newExpr = expr.getExpressionType().getExpressionClass().newInstance();
        } catch (Exception e) {
            e.printStackTrace();
            throw new RuntimeException(e.getMessage(), e);
        }
        newExpr.setExpressionType(expr.getExpressionType());
        if (ExpressionType.COMPARE_EQUAL == expr.getExpressionType()) {
            newExpr.setLeft(expr.getLeft().clone());
            newExpr.setRight(childSubqueryExpr);
            assert (newExpr instanceof ComparisonExpression);
            ((ComparisonExpression) newExpr).setQuantifier(((ComparisonExpression) expr).getQuantifier());
        } else {
            newExpr.setLeft(childSubqueryExpr);
        }
        // Recurse
        newExpr = ParsedUnionStmt.breakUpSetOpSubquery(newExpr);
        if (retval == null) {
            retval = newExpr;
        } else {
            retval = new ConjunctionExpression(conjuctionType, retval, newExpr);
        }
    }
    return retval;
}

Example 95 with AbstractExpression

use of org.voltdb.expressions.AbstractExpression in project voltdb by VoltDB.

the class ParsedUnionStmt method parseOrderColumn.

/**
     * This is a stripped down version of the ParsedSelectStmt.parseOrderColumn. Since the SET ops
     * are not allowed to have aggregate expressions (HAVING, GROUP BY) (except the individual SELECTS)
     * all the logic handling the aggregates is omitted here
     * @param orderByNode
     * @param leftmostSelectChild
     */
private void parseOrderColumn(VoltXMLElement orderByNode, ParsedSelectStmt leftmostSelectChild) {
    ParsedColInfo.ExpressionAdjuster adjuster = new ParsedColInfo.ExpressionAdjuster() {

        @Override
        public AbstractExpression adjust(AbstractExpression expr) {
            // Union itself can't have aggregate expression
            return expr;
        }
    };
    // Get the display columns from the first child
    List<ParsedColInfo> displayColumns = leftmostSelectChild.orderByColumns();
    ParsedColInfo order_col = ParsedColInfo.fromOrderByXml(leftmostSelectChild, orderByNode, adjuster);
    AbstractExpression order_exp = order_col.expression;
    assert (order_exp != null);
    // helps later when trying to determine ORDER BY coverage (for determinism).
    for (ParsedColInfo col : displayColumns) {
        if (col.alias.equals(order_col.alias) || col.expression.equals(order_exp)) {
            col.orderBy = true;
            col.ascending = order_col.ascending;
            order_col.alias = col.alias;
            order_col.columnName = col.columnName;
            order_col.tableName = col.tableName;
            break;
        }
    }
    assert (!(order_exp instanceof ConstantValueExpression));
    assert (!(order_exp instanceof ParameterValueExpression));
    m_orderColumns.add(order_col);
}