Examples with RelationalPlan org.teiid.query.processor.relational.RelationalPlan used on opensource projects

Example 16 with RelationalPlan

use of org.teiid.query.processor.relational.RelationalPlan in project teiid by teiid.

the class TestMaterialization method testManagedMaterializedTransformationInsert.

@Test
public void testManagedMaterializedTransformationInsert() throws Exception {
    // $NON-NLS-1$
    String userSql = "insert into MatTable1 SELECT * FROM ManagedMatView option nocache ManagedMatView";
    QueryMetadataInterface metadata = RealMetadataFactory.exampleMaterializedView();
    AnalysisRecord analysis = new AnalysisRecord(true, DEBUG);
    Command command = helpGetCommand(userSql, metadata, null);
    BasicSourceCapabilities bsc = TestOptimizer.getTypicalCapabilities();
    RelationalPlan plan = (RelationalPlan) TestOptimizer.helpPlanCommand(command, metadata, new DefaultCapabilitiesFinder(bsc), analysis, new String[] { // $NON-NLS-1$
    "SELECT g_0.e1 FROM MatTable.MatTable AS g_0 WHERE mvstatus('MatView', 'ManagedMatView') = 1" }, ComparisonMode.EXACT_COMMAND_STRING);
    bsc.setCapabilitySupport(Capability.INSERT_WITH_QUERYEXPRESSION, true);
    plan = (RelationalPlan) TestOptimizer.helpPlanCommand(command, metadata, new DefaultCapabilitiesFinder(bsc), analysis, new String[] { // $NON-NLS-1$
    "INSERT INTO MatTable1 (e1) SELECT g_0.e1 FROM MatTable.MatTable AS g_0 WHERE mvstatus('MatView', 'ManagedMatView') = 1" }, ComparisonMode.EXACT_COMMAND_STRING);
    Collection<Annotation> annotations = analysis.getAnnotations();
    // $NON-NLS-1$
    assertNotNull("Expected annotations but got none", annotations);
    // $NON-NLS-1$
    assertEquals("Expected catagory mat view", annotations.iterator().next().getCategory(), Annotation.MATERIALIZED_VIEW);
}

Example 17 with RelationalPlan

use of org.teiid.query.processor.relational.RelationalPlan in project teiid by teiid.

the class TestMaterialization method testManagedMaterializedTransformation.

@Test
public void testManagedMaterializedTransformation() throws Exception {
    // $NON-NLS-1$
    String userSql = "SELECT * FROM ManagedMatView";
    QueryMetadataInterface metadata = RealMetadataFactory.exampleMaterializedView();
    AnalysisRecord analysis = new AnalysisRecord(true, DEBUG);
    Command command = helpGetCommand(userSql, metadata, null);
    RelationalPlan plan = (RelationalPlan) TestOptimizer.helpPlanCommand(command, metadata, getGenericFinder(), analysis, new String[] { // $NON-NLS-1$
    "SELECT g_0.e1 FROM MatTable.MatTable AS g_0 WHERE mvstatus('MatView', 'ManagedMatView') = 1" }, ComparisonMode.EXACT_COMMAND_STRING);
    Collection<Annotation> annotations = analysis.getAnnotations();
    // $NON-NLS-1$
    assertNotNull("Expected annotations but got none", annotations);
    // $NON-NLS-1$
    assertEquals("Expected catagory mat view", annotations.iterator().next().getCategory(), Annotation.MATERIALIZED_VIEW);
}

Example 18 with RelationalPlan

use of org.teiid.query.processor.relational.RelationalPlan in project teiid by teiid.

the class RuleMergeCriteria method planMergeJoin.

/**
 * Look for:
 * [NOT] EXISTS ( )
 * IN ( ) / SOME ( )
 *
 * and replace with a semi join
 */
private PlanNode planMergeJoin(PlanNode current, PlanNode root) throws QueryMetadataException, TeiidComponentException {
    float sourceCost = NewCalculateCostUtil.computeCostForTree(current.getFirstChild(), metadata);
    Criteria crit = (Criteria) current.getProperty(NodeConstants.Info.SELECT_CRITERIA);
    PlannedResult plannedResult = findSubquery(crit, true);
    if (plannedResult.query == null) {
        return current;
    }
    if (sourceCost != NewCalculateCostUtil.UNKNOWN_VALUE && sourceCost < RuleChooseDependent.DEFAULT_INDEPENDENT_CARDINALITY && !plannedResult.mergeJoin) {
        // TODO: see if a dependent join applies the other direction
        return current;
    }
    RelationalPlan originalPlan = (RelationalPlan) plannedResult.query.getProcessorPlan();
    Number originalCardinality = originalPlan.getRootNode().getEstimateNodeCardinality();
    if (!plannedResult.mergeJoin && originalCardinality.floatValue() == NewCalculateCostUtil.UNKNOWN_VALUE) {
        // if it's currently unknown, removing criteria won't make it any better
        return current;
    }
    Collection<GroupSymbol> leftGroups = FrameUtil.findJoinSourceNode(current).getGroups();
    if (!planQuery(leftGroups, false, plannedResult)) {
        if (plannedResult.mergeJoin && analysisRecord != null && analysisRecord.recordAnnotations()) {
            // $NON-NLS-1$ //$NON-NLS-2$
            this.analysisRecord.addAnnotation(new Annotation(Annotation.HINTS, "Could not plan as a merge join: " + crit, "ignoring MJ hint", Priority.HIGH));
        }
        return current;
    }
    // check if the child is already ordered.  TODO: see if the ordering is compatible.
    PlanNode childSort = NodeEditor.findNodePreOrder(root, NodeConstants.Types.SORT, NodeConstants.Types.SOURCE | NodeConstants.Types.JOIN);
    if (childSort != null) {
        if (plannedResult.mergeJoin && analysisRecord != null && analysisRecord.recordAnnotations()) {
            // $NON-NLS-1$ //$NON-NLS-2$
            this.analysisRecord.addAnnotation(new Annotation(Annotation.HINTS, "Could not plan as a merge join since the parent join requires a sort: " + crit, "ignoring MJ hint", Priority.HIGH));
        }
        return current;
    }
    // add an order by, which hopefully will get pushed down
    plannedResult.query.setOrderBy(new OrderBy(plannedResult.rightExpressions).clone());
    for (OrderByItem item : plannedResult.query.getOrderBy().getOrderByItems()) {
        int index = plannedResult.query.getProjectedSymbols().indexOf(item.getSymbol());
        if (index >= 0 && !(item.getSymbol() instanceof ElementSymbol)) {
            item.setSymbol((Expression) plannedResult.query.getProjectedSymbols().get(index).clone());
        }
        item.setExpressionPosition(index);
    }
    try {
        // clone the symbols as they may change during planning
        List<Expression> projectedSymbols = LanguageObject.Util.deepClone(plannedResult.query.getProjectedSymbols(), Expression.class);
        // NOTE: we could tap into the relationalplanner at a lower level to get this in a plan node form,
        // the major benefit would be to reuse the dependent join planning logic if possible.
        RelationalPlan subPlan = (RelationalPlan) QueryOptimizer.optimizePlan(plannedResult.query, metadata, idGenerator, capFinder, analysisRecord, context);
        Number planCardinality = subPlan.getRootNode().getEstimateNodeCardinality();
        if (!plannedResult.mergeJoin) {
            // if we don't have a specific hint, then use costing
            if (planCardinality.floatValue() == NewCalculateCostUtil.UNKNOWN_VALUE || planCardinality.floatValue() > 10000000 || (sourceCost == NewCalculateCostUtil.UNKNOWN_VALUE && planCardinality.floatValue() > 1000) || (sourceCost != NewCalculateCostUtil.UNKNOWN_VALUE && sourceCost * originalCardinality.floatValue() < planCardinality.floatValue() / (100 * Math.log(Math.max(4, sourceCost))))) {
                // bail-out if both are unknown or the new plan is too large
                if (analysisRecord != null && analysisRecord.recordDebug()) {
                    // $NON-NLS-1$ //$NON-NLS-2$
                    current.recordDebugAnnotation("cost of merge join plan was not favorable", null, "semi merge join will not be used", analysisRecord, metadata);
                }
                return current;
            }
        }
        // assume dependent
        if ((sourceCost != NewCalculateCostUtil.UNKNOWN_VALUE && planCardinality.floatValue() != NewCalculateCostUtil.UNKNOWN_VALUE && planCardinality.floatValue() < sourceCost / 8) || (sourceCost == NewCalculateCostUtil.UNKNOWN_VALUE && planCardinality.floatValue() <= 1000)) {
            plannedResult.makeInd = true;
        }
        /*if (plannedResult.makeInd 
					&& plannedResult.query.getCorrelatedReferences() == null
					&& !plannedResult.not
					&& plannedResult.leftExpressions.size() == 1) {
            	//TODO: this should just be a dependent criteria node to avoid sorts
            }*/
        // $NON-NLS-1$ //$NON-NLS-2$
        current.recordDebugAnnotation("Conditions met (hint or cost)", null, "Converting to a semi merge join", analysisRecord, metadata);
        PlanNode semiJoin = NodeFactory.getNewNode(NodeConstants.Types.JOIN);
        semiJoin.addGroups(current.getGroups());
        Set<GroupSymbol> groups = GroupsUsedByElementsVisitor.getGroups(plannedResult.rightExpressions);
        semiJoin.addGroups(groups);
        semiJoin.setProperty(NodeConstants.Info.JOIN_STRATEGY, JoinStrategyType.MERGE);
        semiJoin.setProperty(NodeConstants.Info.JOIN_TYPE, plannedResult.not ? JoinType.JOIN_ANTI_SEMI : JoinType.JOIN_SEMI);
        semiJoin.setProperty(NodeConstants.Info.NON_EQUI_JOIN_CRITERIA, plannedResult.nonEquiJoinCriteria);
        List<Criteria> joinCriteria = new ArrayList<Criteria>();
        joinCriteria.addAll(plannedResult.nonEquiJoinCriteria);
        for (int i = 0; i < plannedResult.leftExpressions.size(); i++) {
            joinCriteria.add(new CompareCriteria((Expression) plannedResult.rightExpressions.get(i), CompareCriteria.EQ, (Expression) plannedResult.leftExpressions.get(i)));
        }
        semiJoin.setProperty(NodeConstants.Info.JOIN_CRITERIA, joinCriteria);
        // nested subqueries are possibly being promoted, so they need their references updated
        List<SymbolMap> refMaps = semiJoin.getAllReferences();
        SymbolMap parentRefs = plannedResult.query.getCorrelatedReferences();
        for (SymbolMap refs : refMaps) {
            for (Map.Entry<ElementSymbol, Expression> ref : refs.asUpdatableMap().entrySet()) {
                Expression expr = ref.getValue();
                if (expr instanceof ElementSymbol) {
                    Expression convertedExpr = parentRefs.getMappedExpression((ElementSymbol) expr);
                    if (convertedExpr != null) {
                        ref.setValue(convertedExpr);
                    }
                }
                semiJoin.getGroups().addAll(GroupsUsedByElementsVisitor.getGroups(ref.getValue()));
            }
        }
        semiJoin.setProperty(NodeConstants.Info.LEFT_EXPRESSIONS, plannedResult.leftExpressions);
        semiJoin.getGroups().addAll(GroupsUsedByElementsVisitor.getGroups(plannedResult.leftExpressions));
        semiJoin.setProperty(NodeConstants.Info.RIGHT_EXPRESSIONS, plannedResult.rightExpressions);
        semiJoin.getGroups().addAll(GroupsUsedByElementsVisitor.getGroups(plannedResult.rightExpressions));
        semiJoin.setProperty(NodeConstants.Info.SORT_RIGHT, SortOption.ALREADY_SORTED);
        semiJoin.setProperty(NodeConstants.Info.OUTPUT_COLS, root.getProperty(NodeConstants.Info.OUTPUT_COLS));
        List childOutput = (List) current.getFirstChild().getProperty(NodeConstants.Info.OUTPUT_COLS);
        PlanNode toCorrect = root;
        while (toCorrect != current) {
            toCorrect.setProperty(NodeConstants.Info.OUTPUT_COLS, childOutput);
            toCorrect = toCorrect.getFirstChild();
        }
        PlanNode node = NodeFactory.getNewNode(NodeConstants.Types.ACCESS);
        node.setProperty(NodeConstants.Info.PROCESSOR_PLAN, subPlan);
        node.setProperty(NodeConstants.Info.OUTPUT_COLS, projectedSymbols);
        node.setProperty(NodeConstants.Info.EST_CARDINALITY, planCardinality);
        node.addGroups(groups);
        root.addAsParent(semiJoin);
        semiJoin.addLastChild(node);
        PlanNode result = current.getParent();
        NodeEditor.removeChildNode(result, current);
        RuleImplementJoinStrategy.insertSort(semiJoin.getFirstChild(), (List<Expression>) plannedResult.leftExpressions, semiJoin, metadata, capFinder, true, context);
        if (plannedResult.makeInd && !plannedResult.not) {
            // TODO: would like for an enhanced sort merge with the semi dep option to avoid the sorting
            // this is a little different than a typical dependent join in that the right is the independent side
            String id = RuleChooseDependent.nextId();
            PlanNode dep = RuleChooseDependent.getDependentCriteriaNode(id, plannedResult.rightExpressions, plannedResult.leftExpressions, node, metadata, null, false, null);
            semiJoin.getFirstChild().addAsParent(dep);
            semiJoin.setProperty(NodeConstants.Info.DEPENDENT_VALUE_SOURCE, id);
            this.dependent = true;
        }
        return result;
    } catch (QueryPlannerException e) {
        // can't be done - probably access patterns - what about dependent
        return current;
    }
}

Example 19 with RelationalPlan

use of org.teiid.query.processor.relational.RelationalPlan in project teiid by teiid.

the class RelationalPlanner method planWith.

private void planWith(PlanNode plan, Command command) throws QueryPlannerException, QueryMetadataException, TeiidComponentException, QueryResolverException {
    if (this.withPlanningState.withList.isEmpty()) {
        return;
    }
    // TODO: merge this logic inline with the main rule execution.
    RuleStack stack = new RuleStack();
    stack.push(new RuleAssignOutputElements(false));
    if (hints.hasRowBasedSecurity) {
        stack.push(new RuleApplySecurity());
    }
    // use a temporary planner to run just the assign output elements
    RelationalPlanner planner = new RelationalPlanner();
    // we don't want to trigger the with processing for just projection
    planner.processWith = false;
    planner.initialize(command, idGenerator, metadata, capFinder, analysisRecord, context);
    planner.executeRules(stack, plan);
    // discover all of the usage
    List<Command> commands = CommandCollectorVisitor.getCommands(command, true);
    while (!commands.isEmpty()) {
        Command cmd = commands.remove(commands.size() - 1);
        commands.addAll(CommandCollectorVisitor.getCommands(cmd, true));
        try {
            PlanNode temp = planner.generatePlan((Command) cmd.clone());
            stack.push(new RuleAssignOutputElements(false));
            planner.executeRules(stack, temp);
        } catch (TeiidProcessingException e) {
            throw new QueryPlannerException(e);
        }
    }
    // plan and minimize projection
    for (WithQueryCommand with : this.withPlanningState.withList.values()) {
        QueryCommand subCommand = with.getCommand();
        TempMetadataID tid = (TempMetadataID) with.getGroupSymbol().getMetadataID();
        if (tid.getTableData().getModel() != TempMetadataAdapter.TEMP_MODEL) {
            tid.getTableData().setModel(null);
        }
        List<TempMetadataID> elements = tid.getElements();
        List<Integer> toRemove = new ArrayList<Integer>();
        for (int i = elements.size() - 1; i >= 0; i--) {
            TempMetadataID elem = elements.get(i);
            if (!elem.isAccessed()) {
                toRemove.add(i);
            }
        }
        // the definition of the with clause consistent
        if (!toRemove.isEmpty()) {
            if (with.isRecursive()) {
                SetQuery setQuery = (SetQuery) subCommand;
                setQuery.setLeftQuery(removeUnusedProjection(with, setQuery.getLeftQuery(), elements, toRemove));
                setQuery.setRightQuery(removeUnusedProjection(with, setQuery.getRightQuery(), elements, toRemove));
            } else {
                subCommand = removeUnusedProjection(with, subCommand, elements, toRemove);
                with.setCommand(subCommand);
            }
        }
        if (with.isRecursive()) {
            SetQuery setQuery = (SetQuery) subCommand;
            QueryCommand qc = setQuery.getLeftQuery();
            final RelationalPlan subPlan = optimize(qc);
            qc.setProcessorPlan(subPlan);
            AccessNode aNode = CriteriaCapabilityValidatorVisitor.getAccessNode(subPlan);
            Object modelID = null;
            QueryCommand withCommand = null;
            if (aNode != null) {
                modelID = CriteriaCapabilityValidatorVisitor.validateCommandPushdown(null, metadata, capFinder, aNode, false);
                if (modelID != null) {
                    if (with.getGroupSymbol().getModelMetadataId() != null || !CapabilitiesUtil.supports(Capability.RECURSIVE_COMMON_TABLE_EXPRESSIONS, modelID, metadata, capFinder) || with.isMaterialize()) {
                        modelID = null;
                    } else {
                        withCommand = CriteriaCapabilityValidatorVisitor.getQueryCommand(aNode);
                        if (withCommand != null) {
                            // provisionally set the source
                            ((TempMetadataID) with.getGroupSymbol().getMetadataID()).getTableData().setModel(modelID);
                        }
                    }
                }
            }
            // now that we possibly have a model id, plan the recursive part
            QueryCommand qc1 = setQuery.getRightQuery();
            RelationalPlan subPlan1 = optimize((Command) qc1.clone());
            qc1.setProcessorPlan(subPlan1);
            if (!isPushdownValid(with, setQuery, modelID, withCommand, subPlan1) && withCommand != null) {
                // reset the source to null and replan
                ((TempMetadataID) with.getGroupSymbol().getMetadataID()).getTableData().setModel(null);
                subPlan1 = optimize(qc1);
                qc1.setProcessorPlan(subPlan1);
            }
            continue;
        }
        RelationalPlan subPlan = optimize(subCommand);
        subCommand.setProcessorPlan(subPlan);
        RelationalPlan procPlan = subPlan;
        RelationalNode root = procPlan.getRootNode();
        Number planCardinality = root.getEstimateNodeCardinality();
        if (planCardinality != null) {
            ((TempMetadataID) with.getGroupSymbol().getMetadataID()).setCardinality(planCardinality.intValue());
        }
        AccessNode aNode = CriteriaCapabilityValidatorVisitor.getAccessNode(procPlan);
        if (aNode == null) {
            continue;
        }
        Object modelID = CriteriaCapabilityValidatorVisitor.validateCommandPushdown(null, metadata, capFinder, aNode, false);
        QueryCommand withCommand = CriteriaCapabilityValidatorVisitor.getQueryCommand(aNode);
        if (modelID == null || withCommand == null) {
            continue;
        }
        if (with.getGroupSymbol().getModelMetadataId() != null || !CapabilitiesUtil.supports(Capability.COMMON_TABLE_EXPRESSIONS, modelID, metadata, capFinder) || with.isMaterialize()) {
            continue;
        }
        WithQueryCommand wqc = new WithQueryCommand(with.getGroupSymbol(), with.getColumns(), withCommand);
        wqc.setNoInline(with.isNoInline());
        ((TempMetadataID) with.getGroupSymbol().getMetadataID()).getTableData().setModel(modelID);
        this.withPlanningState.pushdownWith.put(with.getGroupSymbol().getName(), wqc);
    }
}

Example 20 with RelationalPlan

use of org.teiid.query.processor.relational.RelationalPlan in project teiid by teiid.

the class RelationalPlanner method optimize.

public RelationalPlan optimize(Command command) throws QueryPlannerException, QueryMetadataException, TeiidComponentException, QueryResolverException {
    boolean debug = analysisRecord.recordDebug();
    if (debug) {
        // $NON-NLS-1$
        analysisRecord.println("\n----------------------------------------------------------------------------");
        // $NON-NLS-1$
        analysisRecord.println("GENERATE CANONICAL: \n" + command);
    }
    SourceHint previous = this.sourceHint;
    this.sourceHint = SourceHint.combine(previous, command.getSourceHint());
    PlanToProcessConverter planToProcessConverter = new PlanToProcessConverter(metadata, idGenerator, analysisRecord, capFinder, context);
    WithPlanningState saved = this.withPlanningState;
    this.withPlanningState = new WithPlanningState();
    Command original = (Command) command.clone();
    PlanNode plan;
    try {
        plan = generatePlan(command);
    } catch (TeiidProcessingException e) {
        throw new QueryPlannerException(e);
    }
    planWith(plan, command);
    if (plan.getType() == NodeConstants.Types.SOURCE) {
        // this was effectively a rewrite
        return (RelationalPlan) plan.getProperty(Info.PROCESSOR_PLAN);
    }
    if (debug) {
        // $NON-NLS-1$
        analysisRecord.println("\nCANONICAL PLAN: \n" + plan);
    }
    // Connect ProcessorPlan to SubqueryContainer (if any) of SELECT or PROJECT nodes
    // TODO: merge with node creation
    connectSubqueryContainers(plan);
    // Set top column information on top node
    List<Expression> topCols = Util.deepClone(command.getProjectedSymbols(), Expression.class);
    // Build rule set based on hints
    RuleStack rules = buildRules();
    // Run rule-based optimizer
    plan = executeRules(rules, plan);
    RelationalPlan result = planToProcessConverter.convert(plan);
    boolean fullPushdown = false;
    if (!this.withPlanningState.pushdownWith.isEmpty()) {
        AccessNode aNode = CriteriaCapabilityValidatorVisitor.getAccessNode(result);
        if (aNode != null) {
            QueryCommand queryCommand = CriteriaCapabilityValidatorVisitor.getQueryCommand(aNode);
            if (queryCommand != null) {
                fullPushdown = true;
                for (SubqueryContainer<?> container : ValueIteratorProviderCollectorVisitor.getValueIteratorProviders(queryCommand)) {
                    if (container instanceof Evaluatable<?> && ((Evaluatable<?>) container).shouldEvaluate()) {
                        // we could more deeply check, but we'll just assume that the references are needed
                        fullPushdown = false;
                        break;
                    }
                }
            }
        }
        // distribute the appropriate clauses to the pushdowns
        assignWithClause(result.getRootNode(), this.withPlanningState.pushdownWith, false);
        List<String> toReplan = new ArrayList<String>();
        for (Map.Entry<String, Object> entry : this.withPlanningState.pushdownState.entrySet()) {
            if (Boolean.TRUE.equals(entry.getValue())) {
                GroupSymbol gs = this.withPlanningState.pushdownWith.get(entry.getKey()).getGroupSymbol();
                TempMetadataID tmi = (TempMetadataID) gs.getMetadataID();
                tmi.getTableData().setModel(TempMetadataAdapter.TEMP_MODEL);
                toReplan.add(entry.getKey());
            }
        }
        if (!toReplan.isEmpty()) {
            for (WithQueryCommand wqc : this.withPlanningState.withList.values()) {
                this.context.getGroups().remove(wqc.getGroupSymbol().getName());
            }
            this.sourceHint = previous;
            this.withPlanningState = saved;
            if (debug) {
                // $NON-NLS-1$ //$NON-NLS-2$
                analysisRecord.println("\nReplanning due to multiple common table references: " + toReplan + "\n");
            }
            return optimize(original);
        }
    }
    if (!fullPushdown && !this.withPlanningState.withList.isEmpty()) {
        // generally any with item associated with a pushdown will not be needed as we're converting to a source query
        result.setWith(new ArrayList<WithQueryCommand>(this.withPlanningState.withList.values()));
        // assign any with clauses in this subplan
        for (WithQueryCommand wqc : this.withPlanningState.withList.values()) {
            if (wqc.isRecursive()) {
                SetQuery sq = (SetQuery) wqc.getCommand();
                assignWithClause(((RelationalPlan) sq.getLeftQuery().getProcessorPlan()).getRootNode(), this.withPlanningState.pushdownWith, false);
                assignWithClause(((RelationalPlan) sq.getRightQuery().getProcessorPlan()).getRootNode(), this.withPlanningState.pushdownWith, false);
            } else {
                assignWithClause(((RelationalPlan) wqc.getCommand().getProcessorPlan()).getRootNode(), this.withPlanningState.pushdownWith, false);
            }
        }
    }
    result.setOutputElements(topCols);
    this.sourceHint = previous;
    this.withPlanningState = saved;
    return result;
}