Examples with StmtTableScan org.voltdb.planner.parseinfo.StmtTableScan used on opensource projects

Example 1 with StmtTableScan

use of org.voltdb.planner.parseinfo.StmtTableScan in project voltdb by VoltDB.

the class AbstractParsedStmt method parseColumnRefExpression.

/**
     *
     * @param exprNode
     * @return
     */
private AbstractExpression parseColumnRefExpression(VoltXMLElement exprNode) {
    String tableName = exprNode.attributes.get("table");
    if (tableName == null) {
        assert (m_DDLIndexedTable != null);
        tableName = m_DDLIndexedTable.getTypeName();
    }
    assert (tableName != null);
    String tableAlias = exprNode.attributes.get("tablealias");
    if (tableAlias == null) {
        tableAlias = tableName;
    }
    String columnName = exprNode.attributes.get("column");
    String columnAlias = exprNode.attributes.get("alias");
    // Whether or not this column is the coalesced column produced by a join with a
    // USING predicate.
    String usingAttr = exprNode.attributes.get("using");
    boolean isUsingColumn = usingAttr != null ? Boolean.parseBoolean(usingAttr) : false;
    // Use the index produced by HSQL as a way to differentiate columns that have
    // the same name with a single table (which can happen for subqueries containing joins).
    int differentiator = Integer.parseInt(exprNode.attributes.get("index"));
    if (differentiator == -1 && isUsingColumn) {
        for (VoltXMLElement usingElem : exprNode.children) {
            String usingTableAlias = usingElem.attributes.get("tablealias");
            if (usingTableAlias != null && usingTableAlias.equals(tableAlias)) {
                differentiator = Integer.parseInt(usingElem.attributes.get("index"));
            }
        }
    }
    TupleValueExpression tve = new TupleValueExpression(tableName, tableAlias, columnName, columnAlias, -1, differentiator);
    // Collect the unique columns used in the plan for a given scan.
    // Resolve the tve and add it to the scan's cache of referenced columns
    // Get tableScan where this TVE is originated from. In case of the
    // correlated queries it may not be THIS statement but its parent
    StmtTableScan tableScan = resolveStmtTableScanByAlias(tableAlias);
    if (tableScan == null) {
        // from TestVoltCompler.testScalarSubqueriesExpectedFailures.
        throw new PlanningErrorException("Object not found: " + tableAlias);
    }
    AbstractExpression resolvedExpr = tableScan.resolveTVE(tve);
    if (m_stmtId == tableScan.getStatementId()) {
        return resolvedExpr;
    }
    // This is a TVE from the correlated expression
    int paramIdx = NEXT_PARAMETER_ID++;
    ParameterValueExpression pve = new ParameterValueExpression(paramIdx, resolvedExpr);
    m_parameterTveMap.put(paramIdx, resolvedExpr);
    return pve;
}

Example 2 with StmtTableScan

use of org.voltdb.planner.parseinfo.StmtTableScan in project voltdb by VoltDB.

the class AbstractParsedStmt method addTableToStmtCache.

/**
     * Add a table to the statement cache.
     * @param table
     * @param tableAlias
     * @return the cache entry
     */
protected StmtTableScan addTableToStmtCache(Table table, String tableAlias) {
    // Create an index into the query Catalog cache
    StmtTableScan tableScan = m_tableAliasMap.get(tableAlias);
    if (tableScan == null) {
        tableScan = new StmtTargetTableScan(table, tableAlias, m_stmtId);
        m_tableAliasMap.put(tableAlias, tableScan);
    }
    return tableScan;
}

Example 3 with StmtTableScan

use of org.voltdb.planner.parseinfo.StmtTableScan in project voltdb by VoltDB.

the class ReplaceWithIndexLimit method recursivelyApply.

@Override
protected AbstractPlanNode recursivelyApply(AbstractPlanNode plan) {
    assert (plan != null);
    // depth first:
    //     Find AggregatePlanNode with exactly one child
    //     where that child is an AbstractScanPlanNode.
    //     Replace qualifying SeqScanPlanNode with an
    //     IndexScanPlanNode with an inlined LimitPlanNode;
    //     or appending the LimitPlanNode to the existing
    //     qualified IndexScanPlanNode.
    ArrayList<AbstractPlanNode> children = new ArrayList<AbstractPlanNode>();
    for (int i = 0; i < plan.getChildCount(); i++) children.add(plan.getChild(i));
    for (AbstractPlanNode child : children) {
        // TODO this will break when children feed multiple parents
        AbstractPlanNode newChild = recursivelyApply(child);
        // Do a graft into the (parent) plan only if a replacement for a child was found.
        if (newChild == child) {
            continue;
        }
        child.removeFromGraph();
        plan.addAndLinkChild(newChild);
    }
    // check for an aggregation of the right form
    if ((plan instanceof AggregatePlanNode) == false)
        return plan;
    assert (plan.getChildCount() == 1);
    AggregatePlanNode aggplan = (AggregatePlanNode) plan;
    // handle one single min() / max() now
    // TODO: combination of [min(), max(), count()]
    SortDirectionType sortDirection = SortDirectionType.INVALID;
    if (aggplan.isTableMin()) {
        sortDirection = SortDirectionType.ASC;
    } else if (aggplan.isTableMax()) {
        sortDirection = SortDirectionType.DESC;
    } else {
        return plan;
    }
    AbstractPlanNode child = plan.getChild(0);
    AbstractExpression aggExpr = aggplan.getFirstAggregateExpression();
    // for a SEQSCAN, replace it with a INDEXSCAN node with an inline LIMIT plan node
    if (child instanceof SeqScanPlanNode) {
        // should have other index access plan if any qualified index found for the predicate
        if (((SeqScanPlanNode) child).getPredicate() != null) {
            return plan;
        }
        if (((AbstractScanPlanNode) child).isSubQuery()) {
            return plan;
        }
        // create an empty bindingExprs list, used for store (possible) bindings for adHoc query
        ArrayList<AbstractExpression> bindings = new ArrayList<AbstractExpression>();
        Index ret = findQualifiedIndex(((SeqScanPlanNode) child), aggExpr, bindings);
        if (ret == null) {
            return plan;
        } else {
            // 1. create one INDEXSCAN plan node with inlined LIMIT
            // and replace the SEQSCAN node with it
            // 2. we know which end row we want to fetch, so it's safe to
            // specify sorting direction here
            IndexScanPlanNode ispn = new IndexScanPlanNode((SeqScanPlanNode) child, aggplan, ret, sortDirection);
            ispn.setBindings(bindings);
            assert (ispn.getSearchKeyExpressions().size() == 0);
            if (sortDirection == SortDirectionType.ASC) {
                assert (aggplan.isTableMin());
                ispn.setSkipNullPredicate(0);
            }
            LimitPlanNode lpn = new LimitPlanNode();
            lpn.setLimit(1);
            lpn.setOffset(0);
            ispn.addInlinePlanNode(lpn);
            // remove old SeqScan node and link the new generated IndexScan node
            plan.clearChildren();
            plan.addAndLinkChild(ispn);
            return plan;
        }
    }
    if ((child instanceof IndexScanPlanNode) == false) {
        return plan;
    }
    // already have the IndexScanPlanNode
    IndexScanPlanNode ispn = (IndexScanPlanNode) child;
    // we added for reverse scan purpose only
    if (((IndexScanPlanNode) child).getPredicate() != null && !((IndexScanPlanNode) child).isPredicatesOptimizableForAggregate()) {
        return plan;
    }
    // Guard against (possible future?) cases of indexable subquery.
    if (((AbstractScanPlanNode) child).isSubQuery()) {
        return plan;
    }
    // 2. Handle equality filters and one other comparison operator (<, <=, >, >=), see comments below
    if (ispn.getLookupType() != IndexLookupType.EQ && Math.abs(ispn.getSearchKeyExpressions().size() - ExpressionUtil.uncombinePredicate(ispn.getEndExpression()).size()) > 1) {
        return plan;
    }
    // exprs will be used as filterExprs to check the index
    // For forward scan, the initial value is endExprs and might be changed in different values in variant cases
    // For reverse scan, the initial value is initialExprs which is the "old" endExprs
    List<AbstractExpression> exprs;
    int numOfSearchKeys = ispn.getSearchKeyExpressions().size();
    if (ispn.getLookupType() == IndexLookupType.LT || ispn.getLookupType() == IndexLookupType.LTE) {
        exprs = ExpressionUtil.uncombinePredicate(ispn.getInitialExpression());
        numOfSearchKeys -= 1;
    } else {
        exprs = ExpressionUtil.uncombinePredicate(ispn.getEndExpression());
    }
    int numberOfExprs = exprs.size();
    /* Retrieve the index expressions from the target index. (ENG-8819, Ethan)
         * This is because we found that for the following two queries:
         *     #1: explain select max(c2/2) from t where c1=1 and c2/2<=3;
         *     #2: explain select max(c2/2) from t where c1=1 and c2/2<=?;
         * We can get an inline limit 1 for #2 but not for #1. This is because all constants in #1 got parameterized.
         * The result is that the query cannot pass the bindingToIndexedExpression() tests below
         * because we lost all the constant value expressions (cannot attempt to bind a pve to a pve!).
         * Those constant values expressions can only be accessed from the idnex.
         * We will not add those bindings to the ispn.getBindings() here because they will be added anyway in checkIndex().
         * PS: For this case (i.e. index on expressions), checkIndex() will call checkExpressionIndex(),
         * where bindings will be added.
         */
    Index indexToUse = ispn.getCatalogIndex();
    String tableAlias = ispn.getTargetTableAlias();
    List<AbstractExpression> indexedExprs = null;
    if (!indexToUse.getExpressionsjson().isEmpty()) {
        StmtTableScan tableScan = m_parsedStmt.getStmtTableScanByAlias(tableAlias);
        try {
            indexedExprs = AbstractExpression.fromJSONArrayString(indexToUse.getExpressionsjson(), tableScan);
        } catch (JSONException e) {
            e.printStackTrace();
            assert (false);
            return plan;
        }
    }
    /* If there is only 1 difference between searchkeyExprs and endExprs,
         * 1. trivial filters can be discarded, 2 possibilities:
         *      a. SELECT MIN(X) FROM T WHERE [other prefix filters] X < / <= ?
         *         <=> SELECT MIN(X) FROM T WHERE [other prefix filters] && the X < / <= ? filter
         *      b. SELECT MAX(X) FROM T WHERE X > / >= ?
         *         <=> SELECT MAX(X) FROM T with post-filter
         * 2. filter should act as equality filter, 2 possibilities
         *      SELECT MIN(X) FROM T WHERE [other prefix filters] X > / >= ?
         *      SELECT MAX(X) FROM T WHERE [other prefix filters] X < / <= ?

         * check if there is other filters for SELECT MAX(X) FROM T WHERE [other prefix filter AND ] X > / >= ?
         * but we should allow SELECT MAX(X) FROM T WHERE X = ?

         * This is for queries having MAX() but no ORDER BY. (ENG-8819, Ethan)
         * sortDirection == DESC if max, ASC if min. ispn.getSortDirection() == INVALID if no ORDER BY. */
    if (sortDirection == SortDirectionType.DESC && ispn.getSortDirection() == SortDirectionType.INVALID) {
        /* numberOfExprs = exprs.size(), exprs are initial expressions for reversed index scans (lookupType LT, LTE),
             * are end expressions for forward index scans (lookupType GT, GTE, EQ).
             * Note, lookupType doesn't decide the scan direction for sure. MIN(X) where X < ? is still a forward scan.
             * X < ? will be a post filter for the scan rather than an initial expression. */
        if (numberOfExprs == 1) {
            // e.g.: explain select max(c2/2) from t where c2/2<=3;
            // In this case, as long as the where condition (exprs.get(0)) matches the aggregation argument, continue.
            AbstractExpression exprToBind = indexedExprs == null ? exprs.get(0).getLeft() : indexedExprs.get(0);
            if (aggExpr.bindingToIndexedExpression(exprToBind) == null) {
                return plan;
            }
        } else if (numberOfExprs > 1) {
            // ENG-4016: Optimization for query SELECT MAX(X) FROM T WHERE [other prefix filters] X < / <= ?
            // Just keep trying, don't return early.
            boolean earlyReturn = true;
            for (int i = 0; i < numberOfExprs; ++i) {
                AbstractExpression expr = exprs.get(i);
                AbstractExpression indexedExpr = indexedExprs == null ? expr.getLeft() : indexedExprs.get(i);
                if (aggExpr.bindingToIndexedExpression(indexedExpr) != null && (expr.getExpressionType() == ExpressionType.COMPARE_LESSTHANOREQUALTO || expr.getExpressionType() == ExpressionType.COMPARE_LESSTHAN || expr.getExpressionType() == ExpressionType.COMPARE_EQUAL)) {
                    earlyReturn = false;
                    break;
                }
            }
            if (earlyReturn) {
                return plan;
            }
        }
    }
    // have an upper bound: # of endingExpr is more than # of searchExpr
    if (numberOfExprs > numOfSearchKeys) {
        AbstractExpression lastEndExpr = exprs.get(numberOfExprs - 1);
        // check last ending condition, see whether it is
        //      SELECT MIN(X) FROM T WHERE [other prefix filters] X < / <= ? or
        // other filters will be checked later
        AbstractExpression exprToBind = indexedExprs == null ? lastEndExpr.getLeft() : indexedExprs.get(numberOfExprs - 1);
        if ((lastEndExpr.getExpressionType() == ExpressionType.COMPARE_LESSTHAN || lastEndExpr.getExpressionType() == ExpressionType.COMPARE_LESSTHANOREQUALTO) && aggExpr.bindingToIndexedExpression(exprToBind) != null) {
            exprs.remove(lastEndExpr);
        }
    }
    // and we can take advantage of that
    if (checkIndex(ispn.getCatalogIndex(), aggExpr, exprs, ispn.getBindings(), tableAlias)) {
        // we know which end we want to fetch, set the sort direction
        ispn.setSortDirection(sortDirection);
        // for SELECT MIN(X) FROM T WHERE [prefix filters] = ?
        if (numberOfExprs == numOfSearchKeys && sortDirection == SortDirectionType.ASC) {
            if (ispn.getLookupType() == IndexLookupType.GTE) {
                assert (aggplan.isTableMin());
                ispn.setSkipNullPredicate(numOfSearchKeys);
            }
        }
        // reset the IndexLookupType, remove "added" searchKey, add back to endExpression, and clear "added" predicate
        if (sortDirection == SortDirectionType.ASC && (ispn.getLookupType() == IndexLookupType.LT || ispn.getLookupType() == IndexLookupType.LTE)) {
            ispn.setLookupType(IndexLookupType.GTE);
            ispn.removeLastSearchKey();
            ispn.addEndExpression(ExpressionUtil.uncombinePredicate(ispn.getInitialExpression()).get(numberOfExprs - 1));
            ispn.setSkipNullPredicate(numOfSearchKeys);
            ispn.resetPredicate();
        }
        // add an inline LIMIT plan node to this index scan plan node
        LimitPlanNode lpn = new LimitPlanNode();
        lpn.setLimit(1);
        lpn.setOffset(0);
        ispn.addInlinePlanNode(lpn);
        //                                              |__LimitPlanNode
        if (sortDirection == SortDirectionType.DESC && !ispn.getSearchKeyExpressions().isEmpty() && exprs.isEmpty() && ExpressionUtil.uncombinePredicate(ispn.getInitialExpression()).isEmpty()) {
            AbstractExpression newPredicate = new ComparisonExpression();
            if (ispn.getLookupType() == IndexLookupType.GT)
                newPredicate.setExpressionType(ExpressionType.COMPARE_GREATERTHAN);
            if (ispn.getLookupType() == IndexLookupType.GTE)
                newPredicate.setExpressionType(ExpressionType.COMPARE_GREATERTHANOREQUALTO);
            newPredicate.setRight(ispn.getSearchKeyExpressions().get(0));
            newPredicate.setLeft(aggExpr);
            newPredicate.setValueType(aggExpr.getValueType());
            ispn.clearSearchKeyExpression();
            aggplan.setPrePredicate(newPredicate);
        }
    }
    return plan;
}

Example 4 with StmtTableScan

use of org.voltdb.planner.parseinfo.StmtTableScan in project voltdb by VoltDB.

the class StatementPartitioning method analyzeTablePartitioning.

/**
     * This simple analysis counts the number of partitioned tables in the join tree
     * of a query, and initializes a guess for the count of independently partitioned tables.
     *
     * @param tableCacheList
     * @throws PlanningErrorException
     */
void analyzeTablePartitioning(Collection<StmtTableScan> collection) throws PlanningErrorException {
    m_countOfPartitionedTables = 0;
    // Iterate over the tables to collect partition columns.
    for (StmtTableScan tableScan : collection) {
        if (!tableScan.getIsReplicated()) {
            ++m_countOfPartitionedTables;
        }
    }
    // Initial guess -- as if no equality filters.
    m_countOfIndependentlyPartitionedTables = m_countOfPartitionedTables;
}

Example 5 with StmtTableScan

use of org.voltdb.planner.parseinfo.StmtTableScan in project voltdb by VoltDB.

the class StatementPartitioning method analyzeForMultiPartitionAccess.

/**
     * Given the query's list of tables and its collection(s) of equality-filtered columns and their equivalents,
     * determine whether all joins involving partitioned tables can be executed locally on a single partition.
     * This is only the case when they include equality comparisons between partition key columns.
     * VoltDB will reject joins of multiple partitioned tables unless all their partition keys are
     * constrained to be equal to each other.
     * Example: select * from T1, T2 where T1.ID = T2.ID
     * Additionally, in this case, there may be a constant equality filter on any of the columns,
     * which we want to extract as our SP partitioning parameter.
     *
     * @param tableAliasList The tables.
     * @param valueEquivalence Their column equality filters
     * @return the number of independently partitioned tables
     *         -- partitioned tables that aren't joined or filtered by the same value.
     *         The caller can raise an alarm if there is more than one.
     */
public void analyzeForMultiPartitionAccess(Collection<StmtTableScan> scans, HashMap<AbstractExpression, Set<AbstractExpression>> valueEquivalence) {
    //* enable to debug */ System.out.println("DEBUG: analyze4MPAccess w/ scans:" + scans.size() + " filters:" + valueEquivalence.size());
    TupleValueExpression tokenPartitionKey = null;
    Set<Set<AbstractExpression>> eqSets = new HashSet<Set<AbstractExpression>>();
    int unfilteredPartitionKeyCount = 0;
    // reset this flag to forget the last result of the multiple partition access path.
    // AdHoc with parameters will call this function at least two times
    // By default this flag should be true.
    setJoinValid(true);
    setJoinInvalidReason(null);
    boolean subqueryHasReceiveNode = false;
    boolean hasPartitionedTableJoin = false;
    // Iterate over the tables to collect partition columns.
    for (StmtTableScan tableScan : scans) {
        // Replicated tables don't need filter coverage.
        if (tableScan.getIsReplicated()) {
            continue;
        }
        // The partition column can be null in an obscure edge case.
        // The table is declared non-replicated yet specifies no partitioning column.
        // This can occur legitimately when views based on partitioned tables neglect to group by the partition column.
        // The interpretation of this edge case is that the table has "randomly distributed data".
        // In such a case, the table is valid for use by MP queries only and can only be joined with replicated tables
        // because it has no recognized partitioning join key.
        List<SchemaColumn> columnsNeedingCoverage = tableScan.getPartitioningColumns();
        if (tableScan instanceof StmtSubqueryScan) {
            StmtSubqueryScan subScan = (StmtSubqueryScan) tableScan;
            subScan.promoteSinglePartitionInfo(valueEquivalence, eqSets);
            CompiledPlan subqueryPlan = subScan.getBestCostPlan();
            if ((!subScan.canRunInOneFragment()) || ((subqueryPlan != null) && subqueryPlan.rootPlanGraph.hasAnyNodeOfClass(AbstractReceivePlanNode.class))) {
                if (subqueryHasReceiveNode) {
                    // Has found another subquery with receive node on the same level
                    // Not going to support this kind of subquery join with 2 fragment plan.
                    setJoinValid(false);
                    setJoinInvalidReason("This multipartition query is not plannable.  " + "It has a subquery which cannot be single partition.");
                    // Still needs to count the independent partition tables
                    break;
                }
                subqueryHasReceiveNode = true;
                if (subScan.isTableAggregate()) {
                    // Any process based on this subquery should require 1 fragment only.
                    continue;
                }
            } else {
                // this subquery partition table without receive node
                hasPartitionedTableJoin = true;
            }
        } else {
            // This table is a partition table
            hasPartitionedTableJoin = true;
        }
        boolean unfiltered = true;
        for (AbstractExpression candidateColumn : valueEquivalence.keySet()) {
            if (!(candidateColumn instanceof TupleValueExpression)) {
                continue;
            }
            TupleValueExpression candidatePartitionKey = (TupleValueExpression) candidateColumn;
            if (!canCoverPartitioningColumn(candidatePartitionKey, columnsNeedingCoverage)) {
                continue;
            }
            unfiltered = false;
            if (tokenPartitionKey == null) {
                tokenPartitionKey = candidatePartitionKey;
            }
            eqSets.add(valueEquivalence.get(candidatePartitionKey));
        }
        if (unfiltered) {
            ++unfilteredPartitionKeyCount;
        }
    }
    // end for each table StmtTableScan in the collection
    m_countOfIndependentlyPartitionedTables = eqSets.size() + unfilteredPartitionKeyCount;
    //* enable to debug */ System.out.println("DEBUG: analyze4MPAccess found: " + m_countOfIndependentlyPartitionedTables + " = " + eqSets.size() + " + " + unfilteredPartitionKeyCount);
    if (m_countOfIndependentlyPartitionedTables > 1) {
        setJoinValid(false);
        setJoinInvalidReason("This query is not plannable.  " + "The planner cannot guarantee that all rows would be in a single partition.");
    }
    // on outer level. Not going to support this kind of join.
    if (subqueryHasReceiveNode && hasPartitionedTableJoin) {
        setJoinValid(false);
        setJoinInvalidReason("This query is not plannable.  It has a subquery which needs cross-partition access.");
    }
    if ((unfilteredPartitionKeyCount == 0) && (eqSets.size() == 1)) {
        for (Set<AbstractExpression> partitioningValues : eqSets) {
            for (AbstractExpression constExpr : partitioningValues) {
                if (constExpr instanceof TupleValueExpression) {
                    continue;
                }
                VoltType valueType = tokenPartitionKey.getValueType();
                addPartitioningExpression(tokenPartitionKey.getTableName() + '.' + tokenPartitionKey.getColumnName(), constExpr, valueType);
                // Only need one constant value.
                break;
            }
        }
    }
}