Examples with MergeReceivePlanNode org.voltdb.plannodes.MergeReceivePlanNode used on opensource projects

Example 1 with MergeReceivePlanNode

use of org.voltdb.plannodes.MergeReceivePlanNode in project voltdb by VoltDB.

the class InlineOrderByIntoMergeReceive method applyOptimization.

/**
     * For MP queries, the coordinator's OrderBy node can be replaced with
     * a specialized Receive node that merges individual partitions results
     * into a final result set if the partitions result set is sorted
     * in the order matching the ORDER BY order
     *
     * @param orderbyNode - ORDER BY node to optimize
     * @return optimized plan
     */
AbstractPlanNode applyOptimization(OrderByPlanNode orderbyNode) {
    // Find all child RECEIVE nodes. We are not interested in the MERGERECEIVE nodes there
    // because they could only come from subqueries.
    List<AbstractPlanNode> receives = orderbyNode.findAllNodesOfType(PlanNodeType.RECEIVE);
    if (receives.isEmpty()) {
        return orderbyNode;
    }
    assert (receives.size() == 1);
    ReceivePlanNode receive = (ReceivePlanNode) receives.get(0);
    // Make sure that this receive node belongs to the same coordinator fragment that
    // the ORDER BY node does. Alternatively, it could belong to a distributed subquery.
    // Walk up the tree starting at the receive node until we hit either a scan node
    // (distributed subquery) or the original order by node (distributed order by)
    // Collect all nodes that are currently in between ORDER BY and RECEIVE nodes
    // If the optimization is possible, they will be converted to inline nodes of
    // the MERGE RECEIVE node. The expected node types are:
    //      LIMIT, AGGREGATE/PARTIALAGGREGATE/HASHAGGREGATE
    // The HASHAGGREGATE must be convertible to AGGREGATE or PARTIALAGGREGATE for optimization
    // to be applicable.
    // LIMIT can be already inline with ORDER BY node
    AbstractPlanNode limitNode = orderbyNode.getInlinePlanNode(PlanNodeType.LIMIT);
    AbstractPlanNode aggregateNode = null;
    AbstractPlanNode inlineCandidate = receive.getParent(0);
    while (orderbyNode != inlineCandidate) {
        if (inlineCandidate instanceof AbstractScanPlanNode) {
            // it's a subquery
            return orderbyNode;
        }
        PlanNodeType nodeType = inlineCandidate.getPlanNodeType();
        if (nodeType == PlanNodeType.LIMIT && limitNode == null) {
            limitNode = inlineCandidate;
        } else if ((nodeType == PlanNodeType.AGGREGATE || nodeType == PlanNodeType.PARTIALAGGREGATE) && aggregateNode == null) {
            aggregateNode = inlineCandidate;
        } else if (nodeType == PlanNodeType.HASHAGGREGATE && aggregateNode == null) {
            aggregateNode = convertToSerialAggregation(inlineCandidate, orderbyNode);
            if (PlanNodeType.HASHAGGREGATE == aggregateNode.getPlanNodeType()) {
                return orderbyNode;
            }
        } else {
            // Don't know how to handle this node or there is already a node of this type
            return orderbyNode;
        }
        // move up one node
        assert (inlineCandidate.getParentCount() == 1);
        inlineCandidate = inlineCandidate.getParent(0);
    }
    assert (receive.getChildCount() == 1);
    AbstractPlanNode partitionRoot = receive.getChild(0);
    if (!partitionRoot.isOutputOrdered(orderbyNode.getSortExpressions(), orderbyNode.getSortDirections())) {
        // Partition results are not ordered
        return orderbyNode;
    }
    // the new MERGERECIEVE node.. All in-between nodes will be inlined
    assert (orderbyNode.getParentCount() <= 1);
    AbstractPlanNode rootNode = (orderbyNode.getParentCount() == 1) ? orderbyNode.getParent(0) : null;
    MergeReceivePlanNode mergeReceive = new MergeReceivePlanNode();
    assert (receive.getChildCount() == 1);
    mergeReceive.addAndLinkChild(receive.getChild(0));
    receive.removeFromGraph();
    if (rootNode == null) {
        rootNode = mergeReceive;
    } else {
        rootNode.clearChildren();
        rootNode.addAndLinkChild(mergeReceive);
    }
    // Add inline ORDER BY node and remove inline LIMIT node if any
    mergeReceive.addInlinePlanNode(orderbyNode);
    if (limitNode != null) {
        orderbyNode.removeInlinePlanNode(PlanNodeType.LIMIT);
    }
    // Add inline aggregate
    if (aggregateNode != null) {
        if (limitNode != null) {
            // Inline LIMIT with aggregate
            aggregateNode.addInlinePlanNode(limitNode);
        }
        mergeReceive.addInlinePlanNode(aggregateNode);
    }
    // Add LIMIT if it is exist and wasn't inline with aggregate node
    if (limitNode != null && aggregateNode == null) {
        mergeReceive.addInlinePlanNode(limitNode);
    }
    // return the new root
    return rootNode;
}

Example 2 with MergeReceivePlanNode

use of org.voltdb.plannodes.MergeReceivePlanNode in project voltdb by VoltDB.

the class InlineOrderByIntoMergeReceive method applyOptimization.

/**
     * Convert ReceivePlanNodes into MergeReceivePlanNodes when the
     * RECEIVE node's nearest parent is a window function.  We won't
     * have any inline limits or aggregates here, so this is somewhat
     * simpler than the order by case.
     *
     * @param plan
     * @return
     */
private AbstractPlanNode applyOptimization(WindowFunctionPlanNode plan) {
    assert (plan.getChildCount() == 1);
    assert (plan.getChild(0) != null);
    AbstractPlanNode child = plan.getChild(0);
    assert (child != null);
    // an order by node.
    if (!(child instanceof OrderByPlanNode)) {
        return plan;
    }
    OrderByPlanNode onode = (OrderByPlanNode) child;
    child = onode.getChild(0);
    // for this optimization to work.
    if (!(child instanceof ReceivePlanNode)) {
        return plan;
    }
    ReceivePlanNode receiveNode = (ReceivePlanNode) child;
    assert (receiveNode.getChildCount() == 1);
    child = receiveNode.getChild(0);
    // The Receive node needs a send node child.
    assert (child instanceof SendPlanNode);
    SendPlanNode sendNode = (SendPlanNode) child;
    child = sendNode.getChild(0);
    // returns the number in the plan node.
    if (!(child instanceof IndexSortablePlanNode)) {
        return plan;
    }
    IndexSortablePlanNode indexed = (IndexSortablePlanNode) child;
    if (indexed.indexUse().getWindowFunctionUsesIndex() != 0) {
        return plan;
    }
    // Remove the Receive node and the Order by node
    // and replace them with a MergeReceive node.  Leave
    // the order by node inline in the MergeReceive node,
    // since we need it to calculate the merge.
    plan.clearChildren();
    receiveNode.removeFromGraph();
    MergeReceivePlanNode mrnode = new MergeReceivePlanNode();
    mrnode.addInlinePlanNode(onode);
    mrnode.addAndLinkChild(sendNode);
    plan.addAndLinkChild(mrnode);
    return plan;
}

Example 3 with MergeReceivePlanNode

use of org.voltdb.plannodes.MergeReceivePlanNode in project voltdb by VoltDB.

the class TestPlansSubQueries method testPartitionedLimitOffset.

public void testPartitionedLimitOffset() {
    AbstractPlanNode pn;
    List<AbstractPlanNode> planNodes;
    AbstractPlanNode nlpn;
    // Top aggregation node on coordinator
    planNodes = compileToFragments("SELECT -8, T1.NUM " + "FROM SR4 T0, (select RATIO, NUM, DESC from SP4 order by DESC, NUM, RATIO limit 1 offset 1) T1 " + "WHERE (T1.NUM + 5 ) > 44");
    assertEquals(2, planNodes.size());
    pn = planNodes.get(0);
    assertTrue(pn instanceof SendPlanNode);
    pn = pn.getChild(0);
    assertTrue(pn instanceof ProjectionPlanNode);
    nlpn = pn.getChild(0);
    assertTrue(nlpn instanceof NestLoopPlanNode);
    assertEquals(JoinType.INNER, ((NestLoopPlanNode) nlpn).getJoinType());
    pn = nlpn.getChild(1);
    checkPrimaryKeyIndexScan(pn, "SR4");
    pn = nlpn.getChild(0);
    checkSeqScan(pn, "T1", "NUM");
    pn = pn.getChild(0);
    assertTrue(pn instanceof ProjectionPlanNode);
    pn = pn.getChild(0);
    // inline limit with order by
    assertTrue(pn instanceof MergeReceivePlanNode);
    assertNotNull(pn.getInlinePlanNode(PlanNodeType.LIMIT));
    assertNotNull(pn.getInlinePlanNode(PlanNodeType.ORDERBY));
    pn = planNodes.get(1).getChild(0);
    // inline limit with order by
    assertTrue(pn instanceof OrderByPlanNode);
    assertNotNull(pn.getInlinePlanNode(PlanNodeType.LIMIT));
    pn = pn.getChild(0);
    checkPrimaryKeyIndexScan(pn, "SP4");
    planNodes = compileToFragments("SELECT * FROM (SELECT A, C FROM P1 LIMIT 3) T1 " + "where T1.A = 1 ");
    assertEquals(2, planNodes.size());
}

Example 4 with MergeReceivePlanNode

use of org.voltdb.plannodes.MergeReceivePlanNode in project voltdb by VoltDB.

the class TestPlansDistinct method checkDistinctWithGroupbyPlans.

/**
     *
     * @param distinctSQL Group by query with distinct
     * @param groupbySQL Group by query without distinct
     */
protected void checkDistinctWithGroupbyPlans(String distinctSQL, String groupbySQL, boolean limitPushdown) {
    List<AbstractPlanNode> pns1 = compileToFragments(distinctSQL);
    List<AbstractPlanNode> pns2 = compileToFragments(groupbySQL);
    //printExplainPlan(pns1);
    //printExplainPlan(pns2);
    assertTrue(pns1.get(0) instanceof SendPlanNode);
    assertTrue(pns2.get(0) instanceof SendPlanNode);
    AbstractPlanNode apn1, apn2;
    apn1 = pns1.get(0).getChild(0);
    apn2 = pns2.get(0).getChild(0);
    boolean hasTopProjection1 = false;
    if (apn1 instanceof ProjectionPlanNode) {
        apn1 = apn1.getChild(0);
        hasTopProjection1 = true;
    }
    boolean hasTopProjection2 = false;
    if (apn2 instanceof ProjectionPlanNode) {
        apn2 = apn2.getChild(0);
        hasTopProjection2 = true;
    }
    // DISTINCT plan node is rewrote with GROUP BY and adds above the original GROUP BY node
    // there may be another projection node in between for complex aggregation case
    boolean hasOrderby = false, hasLimit = false;
    boolean groupByMergeReceive = false;
    // infer the ORDERBY/LIMIT information from the base line query
    if (apn2 instanceof OrderByPlanNode) {
        hasOrderby = true;
        if (apn2.getInlinePlanNode(PlanNodeType.LIMIT) != null) {
            hasLimit = true;
        }
        apn2 = apn2.getChild(0);
    } else if (apn2 instanceof LimitPlanNode) {
        hasLimit = true;
        apn2 = apn2.getChild(0);
    } else if (apn2 instanceof MergeReceivePlanNode) {
        assertTrue(apn2.getInlinePlanNode(PlanNodeType.ORDERBY) != null);
        hasOrderby = true;
        hasLimit = apn2.getInlinePlanNode(PlanNodeType.LIMIT) != null;
        groupByMergeReceive = true;
    }
    // check the DISTINCT query plan
    boolean distinctMergeReceive = false;
    if (hasOrderby) {
        if (apn1 instanceof OrderByPlanNode) {
            assertTrue(apn1 instanceof OrderByPlanNode);
            if (hasLimit) {
                // check inline limit
                assertNotNull(apn1.getInlinePlanNode(PlanNodeType.LIMIT));
            }
            apn1 = apn1.getChild(0);
        } else if (apn1 instanceof MergeReceivePlanNode) {
            distinctMergeReceive = true;
            assertNotNull(apn1.getInlinePlanNode(PlanNodeType.ORDERBY));
            assertEquals(0, apn1.getChildCount());
        } else {
            fail("The distinctSQL top node is not OrderBy or MergeReceive.");
        }
    } else if (hasLimit) {
        assertTrue(apn1 instanceof LimitPlanNode);
        apn1 = apn1.getChild(0);
    }
    // Check DISTINCT group by plan node
    if (distinctMergeReceive) {
        AbstractPlanNode aggr = AggregatePlanNode.getInlineAggregationNode(apn1);
        assertTrue(aggr instanceof AggregatePlanNode);
        assertEquals(0, ((AggregatePlanNode) aggr).getAggregateTypesSize());
        assertEquals(pns1.get(0).getOutputSchema().getColumns().size(), ((AggregatePlanNode) aggr).getGroupByExpressionsSize());
        if (hasLimit) {
            // check inline limit
            assertNotNull(aggr.getInlinePlanNode(PlanNodeType.LIMIT));
        }
    } else {
        assertTrue(apn1 instanceof HashAggregatePlanNode);
        assertEquals(0, ((HashAggregatePlanNode) apn1).getAggregateTypesSize());
        assertEquals(pns1.get(0).getOutputSchema().getColumns().size(), ((HashAggregatePlanNode) apn1).getGroupByExpressionsSize());
        apn1 = apn1.getChild(0);
    }
    // check projection node for complex aggregation case
    if (apn1 instanceof ProjectionPlanNode) {
        apn1 = apn1.getChild(0);
        assertFalse(hasTopProjection1);
    }
    if (apn2 instanceof ProjectionPlanNode) {
        apn2 = apn2.getChild(0);
        assertFalse(hasTopProjection2);
    }
    // check the rest plan nodes.
    if (distinctMergeReceive == false && groupByMergeReceive == false) {
        assertEquals(apn1.toExplainPlanString(), apn2.toExplainPlanString());
    } else if (distinctMergeReceive == true && groupByMergeReceive == true) {
        // In case of applied MergeReceive optimization the apn1 and apn2 nodes
        // should not have any children
        assertEquals(0, apn1.getChildCount());
        assertEquals(0, apn2.getChildCount());
    }
    // Distributed DISTINCT GROUP BY
    if (pns1.size() > 1) {
        if (!limitPushdown) {
            assertEquals(pns1.get(1).toExplainPlanString(), pns2.get(1).toExplainPlanString());
            return;
        }
        assertTrue(pns1.get(1) instanceof SendPlanNode);
        assertTrue(pns2.get(1) instanceof SendPlanNode);
        apn1 = pns1.get(1).getChild(0);
        apn2 = pns2.get(1).getChild(0);
        // ignore the ORDER BY/LIMIT pushdown plan node
        // because DISTINCT case can not be pushed down
        assertTrue(apn2 instanceof OrderByPlanNode);
        assertNotNull(apn2.getInlinePlanNode(PlanNodeType.LIMIT));
        apn2 = apn2.getChild(0);
        // winners may produce completely different paths.
        if (distinctMergeReceive == false && groupByMergeReceive == false) {
            assertEquals(apn1.toExplainPlanString(), apn2.toExplainPlanString());
        }
    }
}

Example 5 with MergeReceivePlanNode

use of org.voltdb.plannodes.MergeReceivePlanNode in project voltdb by VoltDB.

the class TestPlansLimit method checkInlineLimitWithOrderby.

private void checkInlineLimitWithOrderby(List<AbstractPlanNode> pns, boolean pushdown) {
    AbstractPlanNode p;
    p = pns.get(0).getChild(0);
    assertTrue(p instanceof ProjectionPlanNode);
    p = p.getChild(0);
    if (p instanceof MergeReceivePlanNode) {
        assertNotNull(p.getInlinePlanNode(PlanNodeType.ORDERBY));
        AbstractPlanNode aggr = AggregatePlanNode.getInlineAggregationNode(p);
        if (aggr != null) {
            assertNotNull(aggr.getInlinePlanNode(PlanNodeType.LIMIT));
        }
    } else {
        assertTrue(p instanceof OrderByPlanNode);
        assertNotNull(p.getInlinePlanNode(PlanNodeType.LIMIT));
    }
    if (pushdown) {
        assertEquals(2, pns.size());
        p = pns.get(1).getChild(0);
        assertTrue(p instanceof OrderByPlanNode);
        assertNotNull(p.getInlinePlanNode(PlanNodeType.LIMIT));
    } else if (pns.size() == 2) {
        p = pns.get(1).getChild(0);
        assertFalse(p.toExplainPlanString().toLowerCase().contains("limit"));
    }
}