Examples with StmtTableScan org.voltdb.planner.parseinfo.StmtTableScan used on opensource projects

Example 11 with StmtTableScan

use of org.voltdb.planner.parseinfo.StmtTableScan in project voltdb by VoltDB.

the class PlanAssembler method setupForNewPlans.

/**
     * Clear any old state and get ready to plan a new plan. The next call to
     * getNextPlan() will return the first candidate plan for these parameters.
     *
     */
private void setupForNewPlans(AbstractParsedStmt parsedStmt) {
    m_bestAndOnlyPlanWasGenerated = false;
    m_partitioning.analyzeTablePartitioning(parsedStmt.allScans());
    if (parsedStmt instanceof ParsedUnionStmt) {
        m_parsedUnion = (ParsedUnionStmt) parsedStmt;
        return;
    }
    if (parsedStmt instanceof ParsedSelectStmt) {
        if (tableListIncludesExportOnly(parsedStmt.m_tableList)) {
            throw new PlanningErrorException("Illegal to read a stream.");
        }
        m_parsedSelect = (ParsedSelectStmt) parsedStmt;
        // Simplify the outer join if possible
        if (m_parsedSelect.m_joinTree instanceof BranchNode) {
            if (!m_parsedSelect.hasJoinOrder()) {
                simplifyOuterJoin((BranchNode) m_parsedSelect.m_joinTree);
            }
            // Convert RIGHT joins to the LEFT ones
            ((BranchNode) m_parsedSelect.m_joinTree).toLeftJoin();
        }
        m_subAssembler = new SelectSubPlanAssembler(m_catalogDb, m_parsedSelect, m_partitioning);
        // Process the GROUP BY information, decide whether it is group by the partition column
        if (isPartitionColumnInGroupbyList(m_parsedSelect.groupByColumns())) {
            m_parsedSelect.setHasPartitionColumnInGroupby();
        }
        if (isPartitionColumnInWindowedAggregatePartitionByList()) {
            m_parsedSelect.setHasPartitionColumnInWindowedAggregate();
        }
        return;
    }
    // check that no modification happens to views
    if (tableListIncludesReadOnlyView(parsedStmt.m_tableList)) {
        throw new PlanningErrorException("Illegal to modify a materialized view.");
    }
    m_partitioning.setIsDML();
    // figure out which table we're updating/deleting
    if (parsedStmt instanceof ParsedSwapStmt) {
        assert (parsedStmt.m_tableList.size() == 2);
        if (tableListIncludesExportOnly(parsedStmt.m_tableList)) {
            throw new PlanningErrorException("Illegal to swap a stream.");
        }
        m_parsedSwap = (ParsedSwapStmt) parsedStmt;
        return;
    }
    Table targetTable = parsedStmt.m_tableList.get(0);
    if (targetTable.getIsreplicated()) {
        if (m_partitioning.wasSpecifiedAsSingle() && !m_partitioning.isReplicatedDmlToRunOnAllPartitions()) {
            String msg = "Trying to write to replicated table '" + targetTable.getTypeName() + "' in a single-partition procedure.";
            throw new PlanningErrorException(msg);
        }
    } else if (m_partitioning.wasSpecifiedAsSingle() == false) {
        m_partitioning.setPartitioningColumnForDML(targetTable.getPartitioncolumn());
    }
    if (parsedStmt instanceof ParsedInsertStmt) {
        m_parsedInsert = (ParsedInsertStmt) parsedStmt;
        // The currently handled inserts are too simple to even require a subplan assembler. So, done.
        return;
    }
    if (parsedStmt instanceof ParsedUpdateStmt) {
        if (tableListIncludesExportOnly(parsedStmt.m_tableList)) {
            throw new PlanningErrorException("Illegal to update a stream.");
        }
        m_parsedUpdate = (ParsedUpdateStmt) parsedStmt;
    } else if (parsedStmt instanceof ParsedDeleteStmt) {
        if (tableListIncludesExportOnly(parsedStmt.m_tableList)) {
            throw new PlanningErrorException("Illegal to delete from a stream.");
        }
        m_parsedDelete = (ParsedDeleteStmt) parsedStmt;
    } else {
        throw new RuntimeException("Unknown subclass of AbstractParsedStmt.");
    }
    if (!m_partitioning.wasSpecifiedAsSingle()) {
        //TODO: When updates and deletes can contain joins, this step may have to be
        // deferred so that the valueEquivalence set can be analyzed per join order.
        // This appears to be an unfortunate side effect of how the HSQL interface
        // misleadingly organizes the placement of join/where filters on the statement tree.
        // This throws off the accounting of equivalence join filters until they can be
        // normalized in analyzeJoinFilters, but that normalization process happens on a
        // per-join-order basis, and so, so must this analysis.
        HashMap<AbstractExpression, Set<AbstractExpression>> valueEquivalence = parsedStmt.analyzeValueEquivalence();
        Collection<StmtTableScan> scans = parsedStmt.allScans();
        m_partitioning.analyzeForMultiPartitionAccess(scans, valueEquivalence);
    }
    m_subAssembler = new WriterSubPlanAssembler(m_catalogDb, parsedStmt, m_partitioning);
}

Example 12 with StmtTableScan

use of org.voltdb.planner.parseinfo.StmtTableScan in project voltdb by VoltDB.

the class PlanAssembler method calculateGroupbyColumnsCovered.

private List<Integer> calculateGroupbyColumnsCovered(Index index, String fromTableAlias, List<AbstractExpression> bindings) {
    List<Integer> coveredGroupByColumns = new ArrayList<>();
    List<ParsedColInfo> groupBys = m_parsedSelect.groupByColumns();
    String exprsjson = index.getExpressionsjson();
    if (exprsjson.isEmpty()) {
        List<ColumnRef> indexedColRefs = CatalogUtil.getSortedCatalogItems(index.getColumns(), "index");
        for (int j = 0; j < indexedColRefs.size(); j++) {
            String indexColumnName = indexedColRefs.get(j).getColumn().getName();
            // ignore order of keys in GROUP BY expr
            int ithCovered = 0;
            boolean foundPrefixedColumn = false;
            for (; ithCovered < groupBys.size(); ithCovered++) {
                AbstractExpression gbExpr = groupBys.get(ithCovered).expression;
                if (!(gbExpr instanceof TupleValueExpression)) {
                    continue;
                }
                TupleValueExpression gbTVE = (TupleValueExpression) gbExpr;
                // TVE column index has not been resolved currently
                if (fromTableAlias.equals(gbTVE.getTableAlias()) && indexColumnName.equals(gbTVE.getColumnName())) {
                    foundPrefixedColumn = true;
                    break;
                }
            }
            if (!foundPrefixedColumn) {
                // no prefix match any more
                break;
            }
            coveredGroupByColumns.add(ithCovered);
            if (coveredGroupByColumns.size() == groupBys.size()) {
                // covered all group by columns already
                break;
            }
        }
    } else {
        StmtTableScan fromTableScan = m_parsedSelect.getStmtTableScanByAlias(fromTableAlias);
        // either pure expression index or mix of expressions and simple columns
        List<AbstractExpression> indexedExprs = null;
        try {
            indexedExprs = AbstractExpression.fromJSONArrayString(exprsjson, fromTableScan);
        } catch (JSONException e) {
            e.printStackTrace();
            // This case sounds impossible
            return coveredGroupByColumns;
        }
        for (AbstractExpression indexExpr : indexedExprs) {
            // ignore order of keys in GROUP BY expr
            List<AbstractExpression> binding = null;
            for (int ithCovered = 0; ithCovered < groupBys.size(); ithCovered++) {
                AbstractExpression gbExpr = groupBys.get(ithCovered).expression;
                binding = gbExpr.bindingToIndexedExpression(indexExpr);
                if (binding != null) {
                    bindings.addAll(binding);
                    coveredGroupByColumns.add(ithCovered);
                    break;
                }
            }
            // no prefix match any more or covered all group by columns already
            if (binding == null || coveredGroupByColumns.size() == groupBys.size()) {
                break;
            }
        }
    }
    return coveredGroupByColumns;
}

Example 13 with StmtTableScan

use of org.voltdb.planner.parseinfo.StmtTableScan in project voltdb by VoltDB.

the class SelectSubPlanAssembler method generateInnerAccessPaths.

/**
     * Generate all possible access paths for an inner node in a join.
     * The set of potential index expressions depends whether the inner node can be inlined
     * with the NLIJ or not. In the former case, inner and inner-outer join expressions can
     * be considered for the index access. In the latter, only inner join expressions qualifies.
     *
     * @param parentNode A parent node to the node to generate paths to.
     */
private void generateInnerAccessPaths(BranchNode parentNode) {
    JoinNode innerChildNode = parentNode.getRightNode();
    assert (innerChildNode != null);
    // In case of inner join WHERE and JOIN expressions can be merged
    if (parentNode.getJoinType() == JoinType.INNER) {
        parentNode.m_joinInnerOuterList.addAll(parentNode.m_whereInnerOuterList);
        parentNode.m_whereInnerOuterList.clear();
        parentNode.m_joinInnerList.addAll(parentNode.m_whereInnerList);
        parentNode.m_whereInnerList.clear();
    }
    if (innerChildNode instanceof BranchNode) {
        generateOuterAccessPaths((BranchNode) innerChildNode);
        generateInnerAccessPaths((BranchNode) innerChildNode);
        // The inner node is a join node itself. Only naive access path is possible
        innerChildNode.m_accessPaths.add(getRelevantNaivePath(parentNode.m_joinInnerOuterList, parentNode.m_joinInnerList));
        return;
    }
    // The inner table can have multiple index access paths based on
    // inner and inner-outer join expressions plus the naive one.
    List<AbstractExpression> filterExprs = null;
    List<AbstractExpression> postExprs = null;
    // the inner join expression will effectively filter out inner tuple prior to the NLJ.
    if (parentNode.getJoinType() != JoinType.FULL) {
        filterExprs = parentNode.m_joinInnerList;
    } else {
        postExprs = parentNode.m_joinInnerList;
    }
    StmtTableScan innerTable = innerChildNode.getTableScan();
    assert (innerTable != null);
    innerChildNode.m_accessPaths.addAll(getRelevantAccessPathsForTable(innerTable, parentNode.m_joinInnerOuterList, filterExprs, postExprs));
    // If there are inner expressions AND inner-outer expressions, it could be that there
    // are indexed access paths that use elements of both in the indexing expressions,
    // especially in the case of a compound index.
    // These access paths can not be considered for use with an NLJ because they rely on
    // inner-outer expressions.
    // If there is a possibility that NLIJ will not be an option due to the
    // "special case" processing that puts a send/receive plan between the join node
    // and its inner child node, other access paths need to be considered that use the
    // same indexes as those identified so far but in a simpler, less effective way
    // that does not rely on inner-outer expressions.
    // The following simplistic method of finding these access paths is to force
    // inner-outer expressions to be handled as NLJ-compatible post-filters and repeat
    // the search for access paths.
    // This will typically generate some duplicate access paths, including the naive
    // sequential scan path and any indexed paths that happened to use only the inner
    // expressions.
    // For now, we deal with this redundancy by dropping (and re-generating) all
    // access paths EXCPT those that reference the inner-outer expressions.
    // TODO: implementing access path hash and equality and possibly using a "Set"
    // would allow deduping as new access paths are added OR
    // the simplified access path search process could be based on
    // the existing indexed access paths -- for each access path that "hasInnerOuterIndexExpression"
    // try to generate and add a simpler access path using the same index,
    // this time with the inner-outer expressions used only as non-indexable post-filters.
    // Don't bother generating these redundant or inferior access paths unless there is
    // an inner-outer expression and a chance that NLIJ will be taken out of the running.
    boolean mayNeedInnerSendReceive = (!m_partitioning.wasSpecifiedAsSingle()) && (m_partitioning.getCountOfPartitionedTables() > 0) && (parentNode.getJoinType() != JoinType.INNER) && !innerTable.getIsReplicated();
    if (mayNeedInnerSendReceive && !parentNode.m_joinInnerOuterList.isEmpty()) {
        List<AccessPath> innerOuterAccessPaths = new ArrayList<>();
        for (AccessPath innerAccessPath : innerChildNode.m_accessPaths) {
            if ((innerAccessPath.index != null) && hasInnerOuterIndexExpression(innerChildNode.getTableAlias(), innerAccessPath.indexExprs, innerAccessPath.initialExpr, innerAccessPath.endExprs)) {
                innerOuterAccessPaths.add(innerAccessPath);
            }
        }
        if (parentNode.getJoinType() != JoinType.FULL) {
            filterExprs = parentNode.m_joinInnerList;
            postExprs = parentNode.m_joinInnerOuterList;
        } else {
            // For FULL join type the inner join expressions must be part of the post predicate
            // in order to stay at the join node and not be pushed down to the inner node
            filterExprs = null;
            postExprs = new ArrayList<>(parentNode.m_joinInnerList);
            postExprs.addAll(parentNode.m_joinInnerOuterList);
        }
        Collection<AccessPath> nljAccessPaths = getRelevantAccessPathsForTable(innerTable, null, filterExprs, postExprs);
        innerChildNode.m_accessPaths.clear();
        innerChildNode.m_accessPaths.addAll(nljAccessPaths);
        innerChildNode.m_accessPaths.addAll(innerOuterAccessPaths);
    }
    assert (innerChildNode.m_accessPaths.size() > 0);
}

Example 14 with StmtTableScan

use of org.voltdb.planner.parseinfo.StmtTableScan in project voltdb by VoltDB.

the class SelectSubPlanAssembler method nextPlan.

/**
     * Pull a join order out of the join orders deque, compute all possible plans
     * for that join order, then append them to the computed plans deque.
     */
@Override
protected AbstractPlanNode nextPlan() {
    // or no more plans can be created
    while (m_plans.size() == 0) {
        // get the join order for us to make plans out of
        JoinNode joinTree = m_joinOrders.poll();
        // no more join orders => no more plans to generate
        if (joinTree == null) {
            return null;
        }
        // Analyze join and filter conditions
        joinTree.analyzeJoinExpressions(m_parsedStmt.m_noTableSelectionList);
        // a query that is a little too quirky or complicated.
        if (!m_parsedStmt.m_noTableSelectionList.isEmpty()) {
            throw new PlanningErrorException("Join with filters that do not depend on joined tables is not supported in VoltDB");
        }
        if (!m_partitioning.wasSpecifiedAsSingle()) {
            // Now that analyzeJoinExpressions has done its job of properly categorizing
            // and placing the various filters that the HSQL parser tends to leave in the strangest
            // configuration, this is the first opportunity to analyze WHERE and JOIN filters'
            // effects on statement partitioning.
            // But this causes the analysis to be run based on a particular join order.
            // Which join orders does this analysis actually need to be run on?
            // Can it be run on the first join order and be assumed to hold for all join orders?
            // If there is a join order that fails to generate a single viable plan, is its
            // determination of partitioning (or partitioning failure) always valid for other
            // join orders, or should the analysis be repeated on a viable join order
            // in that case?
            // For now, analyze each join order independently and when an invalid partitioning is
            // detected, skip the plan generation for that particular ordering.
            // If this causes all plans to be skipped, commonly the case, the PlanAssembler
            // should propagate an error message identifying partitioning as the problem.
            HashMap<AbstractExpression, Set<AbstractExpression>> valueEquivalence = joinTree.getAllEquivalenceFilters();
            Collection<StmtTableScan> scans = m_parsedStmt.allScans();
            m_partitioning.analyzeForMultiPartitionAccess(scans, valueEquivalence);
            if (!m_partitioning.isJoinValid()) {
                // The case of more than one independent partitioned table
                // would result in an illegal plan with more than two fragments.
                // Don't throw a planning error here, in case the problem is just with this
                // particular join order, but do leave a hint to the PlanAssembler in case
                // the failure is unanimous -- a common case.
                m_recentErrorMsg = m_partitioning.getJoinInvalidReason();
                // This join order, at least, is not worth trying to plan.
                continue;
            }
        }
        generateMorePlansForJoinTree(joinTree);
    }
    return m_plans.poll();
}

Example 15 with StmtTableScan

use of org.voltdb.planner.parseinfo.StmtTableScan in project voltdb by VoltDB.

the class ScanDeterminizer method recursivelyApply.

private static AbstractPlanNode recursivelyApply(AbstractPlanNode plan) {
    assert (plan != null);
    // depth first:
    //     Find Sequential Scan node.
    //     Replace with any unique tree index scan if possible.
    ArrayList<AbstractPlanNode> children = new ArrayList<AbstractPlanNode>();
    for (int i = 0; i < plan.getChildCount(); i++) {
        children.add(plan.getChild(i));
    }
    for (AbstractPlanNode child : children) {
        // TODO this will break when children feed multiple parents
        AbstractPlanNode newChild = recursivelyApply(child);
        // Do a graft into the (parent) plan only if a replacement for a child was found.
        if (newChild == child) {
            continue;
        }
        boolean replaced = plan.replaceChild(child, newChild);
        assert (replaced);
    }
    // skip the meat if this isn't a scan node
    if (!(plan instanceof SeqScanPlanNode)) {
        return plan;
    }
    SeqScanPlanNode scanNode = (SeqScanPlanNode) plan;
    if (scanNode.isSubQuery()) {
        // This is a sub-query and can't have indexes
        return plan;
    }
    // got here? we're got ourselves a sequential scan over a real table
    assert (scanNode.getChildCount() == 0);
    StmtTableScan tableScan = scanNode.getTableScan();
    assert (tableScan != null);
    Index indexToScan = null;
    // does anything for performance at all.
    for (Index index : tableScan.getIndexes()) {
        // skip non-unique indexes
        if (index.getUnique() == false) {
            continue;
        } else // skip hash indexes
        if (index.getType() != IndexType.BALANCED_TREE.getValue()) {
            continue;
        } else // skip partial indexes
        if (!index.getPredicatejson().isEmpty()) {
            continue;
        } else if (indexToScan == null || CatalogUtil.getCatalogIndexSize(indexToScan) > CatalogUtil.getCatalogIndexSize(index)) {
            indexToScan = index;
        }
    }
    if (indexToScan == null) {
        return plan;
    }
    // make an index node from the scan node
    IndexScanPlanNode indexScanNode = new IndexScanPlanNode(scanNode, null, indexToScan, SortDirectionType.ASC);
    indexScanNode.setForDeterminismOnly();
    return indexScanNode;
}