use of org.teiid.query.metadata.TempMetadataID in project teiid by teiid.
the class RuleAssignOutputElements method assignOutputElements.
/**
* <p>Assign the output elements at a particular node and recurse the tree. The
* outputElements needed from above the node have been collected in
* outputElements.</p>
*
* <p>SOURCE nodes: If we find a SOURCE node, this must define the top
* of a virtual group. Physical groups can be identified by ACCESS nodes
* at this point in the planning stage. So, we filter the virtual elements
* in the virtual source based on the required output elements.</p>
*
* <p>SET_OP nodes: If we hit a SET_OP node, this must be a union. Unions
* require a lot of special care. Unions have many branches and the projected
* elements in each branch are "equivalent" in terms of nodes above the union.
* This means that any filtering must occur in an identical way in all branches
* of a union.</p>
*
* @param root Node to assign
* @param outputElements Output elements needed for this node
* @param metadata Metadata implementation
*/
private void assignOutputElements(PlanNode root, List<Expression> outputElements, QueryMetadataInterface metadata, CapabilitiesFinder capFinder, RuleStack rules, AnalysisRecord analysisRecord, CommandContext context) throws QueryPlannerException, QueryMetadataException, TeiidComponentException {
int nodeType = root.getType();
// Update this node's output columns based on parent's columns
List<Expression> oldOutput = (List<Expression>) root.setProperty(NodeConstants.Info.OUTPUT_COLS, outputElements);
if (root.getChildCount() == 0) {
// update temp access
if (root.getType() == NodeConstants.Types.SOURCE && root.getGroups().size() == 1) {
GroupSymbol gs = root.getGroups().iterator().next();
if (gs.getMetadataID() instanceof TempMetadataID) {
for (Expression ex : outputElements) {
if (ex instanceof ElementSymbol) {
Object id = ((ElementSymbol) ex).getMetadataID();
if (id instanceof TempMetadataID) {
((TempMetadataID) id).setAccessed(true);
}
}
}
}
}
return;
}
switch(nodeType) {
case NodeConstants.Types.ACCESS:
Command command = FrameUtil.getNonQueryCommand(root);
if (command instanceof StoredProcedure) {
// if the access node represents a stored procedure, then we can't actually change the output symbols
root.setProperty(NodeConstants.Info.OUTPUT_COLS, command.getProjectedSymbols());
} else {
ProcessorPlan plan = FrameUtil.getNestedPlan(root);
if (plan != null && (command == null || !RelationalNodeUtil.isUpdate(command))) {
// nested with clauses are handled as sub plans, which have a fixed set of output symbols
root.setProperty(NodeConstants.Info.OUTPUT_COLS, ResolverUtil.resolveElementsInGroup(root.getGroups().iterator().next(), metadata));
}
if (checkSymbols) {
Object modelId = RuleRaiseAccess.getModelIDFromAccess(root, metadata);
for (Expression symbol : outputElements) {
if (!RuleRaiseAccess.canPushSymbol(symbol, true, modelId, metadata, capFinder, analysisRecord)) {
throw new QueryPlannerException(QueryPlugin.Event.TEIID30258, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30258, symbol, modelId));
}
}
}
if (NodeEditor.findParent(root, NodeConstants.Types.PROJECT, NodeConstants.Types.SOURCE) != null) {
// there's a chance that partial projection was used. we are not a defacto project node
// take credit for creating anything that is not an element symbol
LinkedHashSet<Expression> filteredElements = new LinkedHashSet<Expression>();
for (Expression element : outputElements) {
if (element instanceof ElementSymbol) {
filteredElements.add(element);
} else {
filteredElements.addAll(ElementCollectorVisitor.getElements(element, false));
}
}
outputElements = new ArrayList<Expression>(filteredElements);
}
}
assignOutputElements(root.getLastChild(), outputElements, metadata, capFinder, rules, analysisRecord, context);
break;
case NodeConstants.Types.DUP_REMOVE:
// targeted optimization based upon swapping the dup remove for a limit 1
// TODO: may need to also check for grouping over constants
boolean allConstants = true;
for (Expression ex : outputElements) {
if (!(EvaluatableVisitor.willBecomeConstant(SymbolMap.getExpression(ex)))) {
allConstants = false;
break;
}
}
if (allConstants && addLimit(rules, root, metadata, capFinder)) {
// TODO we could more gracefully handle the !addLimit case
PlanNode parent = root.getParent();
if (parent != null) {
NodeEditor.removeChildNode(root.getParent(), root);
execute(parent, metadata, capFinder, rules, analysisRecord, context);
return;
}
}
case NodeConstants.Types.SORT:
// correct expression positions and update the unrelated flag
OrderBy order = (OrderBy) root.getProperty(NodeConstants.Info.SORT_ORDER);
if (order != null && (oldOutput == null || !oldOutput.equals(outputElements))) {
outputElements = new ArrayList<Expression>(outputElements);
boolean hasUnrelated = false;
for (OrderByItem item : order.getOrderByItems()) {
int index = outputElements.indexOf(item.getSymbol());
if (index != -1) {
item.setExpressionPosition(index);
} else {
hasUnrelated = true;
outputElements.add(item.getSymbol());
}
}
if (!hasUnrelated) {
root.setProperty(NodeConstants.Info.UNRELATED_SORT, false);
} else {
root.setProperty(NodeConstants.Info.UNRELATED_SORT, true);
}
}
assignOutputElements(root.getLastChild(), outputElements, metadata, capFinder, rules, analysisRecord, context);
break;
case NodeConstants.Types.TUPLE_LIMIT:
assignOutputElements(root.getLastChild(), outputElements, metadata, capFinder, rules, analysisRecord, context);
break;
case NodeConstants.Types.SOURCE:
{
outputElements = (List<Expression>) determineSourceOutput(root, outputElements, metadata, capFinder);
if (!finalRun && root.getProperty(Info.PARTITION_INFO) != null && NodeEditor.findParent(root, NodeConstants.Types.JOIN) != null) {
GroupSymbol group = root.getGroups().iterator().next();
Object modelId = RuleDecomposeJoin.getEffectiveModelId(metadata, group);
String name = metadata.getExtensionProperty(modelId, RuleDecomposeJoin.IMPLICIT_PARTITION_COLUMN_NAME, true);
if (name != null) {
// keep projecting the implicit partitioning column through the source so that it can
// be used in the decomposition logic
ElementSymbol es = new ElementSymbol(name, group);
if (!outputElements.contains(es)) {
es.setMetadataID(metadata.getElementID(es.getName()));
outputElements.add(es);
}
}
}
root.setProperty(NodeConstants.Info.OUTPUT_COLS, outputElements);
List<Expression> childElements = filterVirtualElements(root, outputElements, metadata);
assignOutputElements(root.getFirstChild(), childElements, metadata, capFinder, rules, analysisRecord, context);
break;
}
case NodeConstants.Types.SET_OP:
{
for (PlanNode childNode : root.getChildren()) {
PlanNode projectNode = NodeEditor.findNodePreOrder(childNode, NodeConstants.Types.PROJECT);
List<Expression> projectCols = (List<Expression>) projectNode.getProperty(NodeConstants.Info.PROJECT_COLS);
assignOutputElements(childNode, projectCols, metadata, capFinder, rules, analysisRecord, context);
}
break;
}
default:
{
PlanNode sortNode = null;
if (root.getType() == NodeConstants.Types.PROJECT) {
GroupSymbol intoGroup = (GroupSymbol) root.getProperty(NodeConstants.Info.INTO_GROUP);
if (intoGroup != null) {
// if this is a project into, treat the nodes under the source as a new plan root
PlanNode intoRoot = NodeEditor.findNodePreOrder(root, NodeConstants.Types.SOURCE);
execute(intoRoot.getFirstChild(), metadata, capFinder, rules, analysisRecord, context);
return;
}
List<Expression> projectCols = outputElements;
sortNode = NodeEditor.findParent(root, NodeConstants.Types.SORT, NodeConstants.Types.SOURCE);
if (finalRun && sortNode != null && sortNode.hasBooleanProperty(NodeConstants.Info.UNRELATED_SORT)) {
root.getGroups().clear();
root.addGroups(GroupsUsedByElementsVisitor.getGroups(projectCols));
root.addGroups(GroupsUsedByElementsVisitor.getGroups(root.getCorrelatedReferenceElements()));
}
root.setProperty(NodeConstants.Info.PROJECT_COLS, projectCols);
if (root.hasBooleanProperty(Info.HAS_WINDOW_FUNCTIONS)) {
Set<WindowFunction> windowFunctions = getWindowFunctions(projectCols);
if (windowFunctions.isEmpty()) {
root.setProperty(Info.HAS_WINDOW_FUNCTIONS, false);
}
}
}
List<Expression> requiredInput = collectRequiredInputSymbols(root, metadata, capFinder);
// targeted optimization for unnecessary aggregation
if (root.getType() == NodeConstants.Types.GROUP && root.hasBooleanProperty(Info.IS_OPTIONAL) && NodeEditor.findParent(root, NodeConstants.Types.ACCESS) == null) {
PlanNode parent = removeGroupBy(root, metadata);
if (!root.hasCollectionProperty(Info.GROUP_COLS)) {
// just lob off everything under the projection
PlanNode project = NodeEditor.findNodePreOrder(parent, NodeConstants.Types.PROJECT);
project.removeAllChildren();
} else if (!addLimit(rules, parent, metadata, capFinder)) {
// $NON-NLS-1$
throw new AssertionError("expected limit node to be added");
}
execute(parent, metadata, capFinder, rules, analysisRecord, context);
return;
}
// Call children recursively
if (root.getChildCount() == 1) {
assignOutputElements(root.getLastChild(), requiredInput, metadata, capFinder, rules, analysisRecord, context);
if (!finalRun && root.getType() == NodeConstants.Types.PROJECT && sortNode != null && sortNode.hasBooleanProperty(NodeConstants.Info.UNRELATED_SORT)) {
// if this is the initial rule run, remove unrelated order to preserve the original projection
OrderBy elements = (OrderBy) sortNode.getProperty(NodeConstants.Info.SORT_ORDER);
outputElements = new ArrayList<Expression>(outputElements);
for (OrderByItem item : elements.getOrderByItems()) {
if (item.getExpressionPosition() == -1) {
outputElements.remove(item.getSymbol());
}
}
root.setProperty(NodeConstants.Info.PROJECT_COLS, outputElements);
}
} else {
// determine which elements go to each side of the join
for (PlanNode childNode : root.getChildren()) {
Set<GroupSymbol> filterGroups = FrameUtil.findJoinSourceNode(childNode).getGroups();
List<Expression> filteredElements = filterElements(requiredInput, filterGroups);
// Call child recursively
assignOutputElements(childNode, filteredElements, metadata, capFinder, rules, analysisRecord, context);
}
}
}
}
}
use of org.teiid.query.metadata.TempMetadataID in project teiid by teiid.
the class PlanToProcessConverter method setRoutingName.
private void setRoutingName(AccessNode accessNode, PlanNode node, Command command) throws QueryPlannerException, TeiidComponentException {
// Look up connector binding name
try {
Object modelID = node.getProperty(NodeConstants.Info.MODEL_ID);
if (modelID == null || modelID instanceof TempMetadataID) {
if (command instanceof StoredProcedure) {
modelID = ((StoredProcedure) command).getModelID();
} else if (!(command instanceof Create || command instanceof Drop)) {
Collection<GroupSymbol> groups = GroupCollectorVisitor.getGroups(command, true);
GroupSymbol group = groups.iterator().next();
modelID = metadata.getModelID(group.getMetadataID());
}
}
String cbName = metadata.getFullName(modelID);
accessNode.setModelName(cbName);
accessNode.setModelId(modelID);
accessNode.setConformedTo((Set<Object>) node.getProperty(Info.CONFORMED_SOURCES));
} catch (QueryMetadataException e) {
throw new QueryPlannerException(QueryPlugin.Event.TEIID30251, e, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30251));
}
}
use of org.teiid.query.metadata.TempMetadataID in project teiid by teiid.
the class RelationalPlanner method planWith.
private void planWith(PlanNode plan, Command command) throws QueryPlannerException, QueryMetadataException, TeiidComponentException, QueryResolverException {
if (this.withPlanningState.withList.isEmpty()) {
return;
}
// TODO: merge this logic inline with the main rule execution.
RuleStack stack = new RuleStack();
stack.push(new RuleAssignOutputElements(false));
if (hints.hasRowBasedSecurity) {
stack.push(new RuleApplySecurity());
}
// use a temporary planner to run just the assign output elements
RelationalPlanner planner = new RelationalPlanner();
// we don't want to trigger the with processing for just projection
planner.processWith = false;
planner.initialize(command, idGenerator, metadata, capFinder, analysisRecord, context);
planner.executeRules(stack, plan);
// discover all of the usage
List<Command> commands = CommandCollectorVisitor.getCommands(command, true);
while (!commands.isEmpty()) {
Command cmd = commands.remove(commands.size() - 1);
commands.addAll(CommandCollectorVisitor.getCommands(cmd, true));
try {
PlanNode temp = planner.generatePlan((Command) cmd.clone());
stack.push(new RuleAssignOutputElements(false));
planner.executeRules(stack, temp);
} catch (TeiidProcessingException e) {
throw new QueryPlannerException(e);
}
}
// plan and minimize projection
for (WithQueryCommand with : this.withPlanningState.withList.values()) {
QueryCommand subCommand = with.getCommand();
TempMetadataID tid = (TempMetadataID) with.getGroupSymbol().getMetadataID();
if (tid.getTableData().getModel() != TempMetadataAdapter.TEMP_MODEL) {
tid.getTableData().setModel(null);
}
List<TempMetadataID> elements = tid.getElements();
List<Integer> toRemove = new ArrayList<Integer>();
for (int i = elements.size() - 1; i >= 0; i--) {
TempMetadataID elem = elements.get(i);
if (!elem.isAccessed()) {
toRemove.add(i);
}
}
// the definition of the with clause consistent
if (!toRemove.isEmpty()) {
if (with.isRecursive()) {
SetQuery setQuery = (SetQuery) subCommand;
setQuery.setLeftQuery(removeUnusedProjection(with, setQuery.getLeftQuery(), elements, toRemove));
setQuery.setRightQuery(removeUnusedProjection(with, setQuery.getRightQuery(), elements, toRemove));
} else {
subCommand = removeUnusedProjection(with, subCommand, elements, toRemove);
with.setCommand(subCommand);
}
}
if (with.isRecursive()) {
SetQuery setQuery = (SetQuery) subCommand;
QueryCommand qc = setQuery.getLeftQuery();
final RelationalPlan subPlan = optimize(qc);
qc.setProcessorPlan(subPlan);
AccessNode aNode = CriteriaCapabilityValidatorVisitor.getAccessNode(subPlan);
Object modelID = null;
QueryCommand withCommand = null;
if (aNode != null) {
modelID = CriteriaCapabilityValidatorVisitor.validateCommandPushdown(null, metadata, capFinder, aNode, false);
if (modelID != null) {
if (with.getGroupSymbol().getModelMetadataId() != null || !CapabilitiesUtil.supports(Capability.RECURSIVE_COMMON_TABLE_EXPRESSIONS, modelID, metadata, capFinder) || with.isMaterialize()) {
modelID = null;
} else {
withCommand = CriteriaCapabilityValidatorVisitor.getQueryCommand(aNode);
if (withCommand != null) {
// provisionally set the source
((TempMetadataID) with.getGroupSymbol().getMetadataID()).getTableData().setModel(modelID);
}
}
}
}
// now that we possibly have a model id, plan the recursive part
QueryCommand qc1 = setQuery.getRightQuery();
RelationalPlan subPlan1 = optimize((Command) qc1.clone());
qc1.setProcessorPlan(subPlan1);
if (!isPushdownValid(with, setQuery, modelID, withCommand, subPlan1) && withCommand != null) {
// reset the source to null and replan
((TempMetadataID) with.getGroupSymbol().getMetadataID()).getTableData().setModel(null);
subPlan1 = optimize(qc1);
qc1.setProcessorPlan(subPlan1);
}
continue;
}
RelationalPlan subPlan = optimize(subCommand);
subCommand.setProcessorPlan(subPlan);
RelationalPlan procPlan = subPlan;
RelationalNode root = procPlan.getRootNode();
Number planCardinality = root.getEstimateNodeCardinality();
if (planCardinality != null) {
((TempMetadataID) with.getGroupSymbol().getMetadataID()).setCardinality(planCardinality.intValue());
}
AccessNode aNode = CriteriaCapabilityValidatorVisitor.getAccessNode(procPlan);
if (aNode == null) {
continue;
}
Object modelID = CriteriaCapabilityValidatorVisitor.validateCommandPushdown(null, metadata, capFinder, aNode, false);
QueryCommand withCommand = CriteriaCapabilityValidatorVisitor.getQueryCommand(aNode);
if (modelID == null || withCommand == null) {
continue;
}
if (with.getGroupSymbol().getModelMetadataId() != null || !CapabilitiesUtil.supports(Capability.COMMON_TABLE_EXPRESSIONS, modelID, metadata, capFinder) || with.isMaterialize()) {
continue;
}
WithQueryCommand wqc = new WithQueryCommand(with.getGroupSymbol(), with.getColumns(), withCommand);
wqc.setNoInline(with.isNoInline());
((TempMetadataID) with.getGroupSymbol().getMetadataID()).getTableData().setModel(modelID);
this.withPlanningState.pushdownWith.put(with.getGroupSymbol().getName(), wqc);
}
}
use of org.teiid.query.metadata.TempMetadataID in project teiid by teiid.
the class RelationalPlanner method optimize.
public RelationalPlan optimize(Command command) throws QueryPlannerException, QueryMetadataException, TeiidComponentException, QueryResolverException {
boolean debug = analysisRecord.recordDebug();
if (debug) {
// $NON-NLS-1$
analysisRecord.println("\n----------------------------------------------------------------------------");
// $NON-NLS-1$
analysisRecord.println("GENERATE CANONICAL: \n" + command);
}
SourceHint previous = this.sourceHint;
this.sourceHint = SourceHint.combine(previous, command.getSourceHint());
PlanToProcessConverter planToProcessConverter = new PlanToProcessConverter(metadata, idGenerator, analysisRecord, capFinder, context);
WithPlanningState saved = this.withPlanningState;
this.withPlanningState = new WithPlanningState();
Command original = (Command) command.clone();
PlanNode plan;
try {
plan = generatePlan(command);
} catch (TeiidProcessingException e) {
throw new QueryPlannerException(e);
}
planWith(plan, command);
if (plan.getType() == NodeConstants.Types.SOURCE) {
// this was effectively a rewrite
return (RelationalPlan) plan.getProperty(Info.PROCESSOR_PLAN);
}
if (debug) {
// $NON-NLS-1$
analysisRecord.println("\nCANONICAL PLAN: \n" + plan);
}
// Connect ProcessorPlan to SubqueryContainer (if any) of SELECT or PROJECT nodes
// TODO: merge with node creation
connectSubqueryContainers(plan);
// Set top column information on top node
List<Expression> topCols = Util.deepClone(command.getProjectedSymbols(), Expression.class);
// Build rule set based on hints
RuleStack rules = buildRules();
// Run rule-based optimizer
plan = executeRules(rules, plan);
RelationalPlan result = planToProcessConverter.convert(plan);
boolean fullPushdown = false;
if (!this.withPlanningState.pushdownWith.isEmpty()) {
AccessNode aNode = CriteriaCapabilityValidatorVisitor.getAccessNode(result);
if (aNode != null) {
QueryCommand queryCommand = CriteriaCapabilityValidatorVisitor.getQueryCommand(aNode);
if (queryCommand != null) {
fullPushdown = true;
for (SubqueryContainer<?> container : ValueIteratorProviderCollectorVisitor.getValueIteratorProviders(queryCommand)) {
if (container instanceof Evaluatable<?> && ((Evaluatable<?>) container).shouldEvaluate()) {
// we could more deeply check, but we'll just assume that the references are needed
fullPushdown = false;
break;
}
}
}
}
// distribute the appropriate clauses to the pushdowns
assignWithClause(result.getRootNode(), this.withPlanningState.pushdownWith, false);
List<String> toReplan = new ArrayList<String>();
for (Map.Entry<String, Object> entry : this.withPlanningState.pushdownState.entrySet()) {
if (Boolean.TRUE.equals(entry.getValue())) {
GroupSymbol gs = this.withPlanningState.pushdownWith.get(entry.getKey()).getGroupSymbol();
TempMetadataID tmi = (TempMetadataID) gs.getMetadataID();
tmi.getTableData().setModel(TempMetadataAdapter.TEMP_MODEL);
toReplan.add(entry.getKey());
}
}
if (!toReplan.isEmpty()) {
for (WithQueryCommand wqc : this.withPlanningState.withList.values()) {
this.context.getGroups().remove(wqc.getGroupSymbol().getName());
}
this.sourceHint = previous;
this.withPlanningState = saved;
if (debug) {
// $NON-NLS-1$ //$NON-NLS-2$
analysisRecord.println("\nReplanning due to multiple common table references: " + toReplan + "\n");
}
return optimize(original);
}
}
if (!fullPushdown && !this.withPlanningState.withList.isEmpty()) {
// generally any with item associated with a pushdown will not be needed as we're converting to a source query
result.setWith(new ArrayList<WithQueryCommand>(this.withPlanningState.withList.values()));
// assign any with clauses in this subplan
for (WithQueryCommand wqc : this.withPlanningState.withList.values()) {
if (wqc.isRecursive()) {
SetQuery sq = (SetQuery) wqc.getCommand();
assignWithClause(((RelationalPlan) sq.getLeftQuery().getProcessorPlan()).getRootNode(), this.withPlanningState.pushdownWith, false);
assignWithClause(((RelationalPlan) sq.getRightQuery().getProcessorPlan()).getRootNode(), this.withPlanningState.pushdownWith, false);
} else {
assignWithClause(((RelationalPlan) wqc.getCommand().getProcessorPlan()).getRootNode(), this.withPlanningState.pushdownWith, false);
}
}
}
result.setOutputElements(topCols);
this.sourceHint = previous;
this.withPlanningState = saved;
return result;
}
use of org.teiid.query.metadata.TempMetadataID in project teiid by teiid.
the class RelationalPlanner method resolveVirtualGroup.
private Command resolveVirtualGroup(GroupSymbol virtualGroup) throws QueryMetadataException, TeiidComponentException, TeiidProcessingException {
QueryNode qnode = null;
Object metadataID = virtualGroup.getMetadataID();
boolean noCache = isNoCacheGroup(metadata, metadataID, option);
boolean isMaterializedGroup = metadata.hasMaterialization(metadataID);
String cacheString = SQLConstants.Reserved.SELECT;
if (isMaterializedGroup) {
Object matMetadataId = metadata.getMaterialization(metadataID);
String matTableName = null;
CacheHint hint = null;
boolean isImplicitGlobal = matMetadataId == null;
if (isImplicitGlobal) {
TempMetadataID tid = context.getGlobalTableStore().getGlobalTempTableMetadataId(metadataID);
matTableName = tid.getID();
hint = tid.getCacheHint();
matMetadataId = tid;
} else {
matTableName = metadata.getFullName(matMetadataId);
}
if (noCache) {
// not use cache
qnode = metadata.getVirtualPlan(metadataID);
// TODO: update the table for defaultMat
// $NON-NLS-1$
recordAnnotation(analysisRecord, Annotation.MATERIALIZED_VIEW, Priority.LOW, "SimpleQueryResolver.materialized_table_not_used", virtualGroup, matTableName);
} else {
this.context.accessedPlanningObject(matMetadataId);
qnode = new QueryNode(null);
List<ElementSymbol> symbols = new ArrayList<ElementSymbol>();
for (ElementSymbol el : ResolverUtil.resolveElementsInGroup(virtualGroup, metadata)) {
symbols.add(new ElementSymbol(el.getShortName()));
}
Query query = createMatViewQuery(metadataID, matMetadataId, matTableName, symbols, isImplicitGlobal);
query.setCacheHint(hint);
qnode.setCommand(query);
// $NON-NLS-1$
cacheString = "matview";
// $NON-NLS-1$
recordAnnotation(analysisRecord, Annotation.MATERIALIZED_VIEW, Priority.LOW, "SimpleQueryResolver.Query_was_redirected_to_Mat_table", virtualGroup, matTableName);
}
} else {
// Not a materialized view - query the primary transformation
qnode = metadata.getVirtualPlan(metadataID);
}
Command result = (Command) QueryResolver.resolveView(virtualGroup, qnode, cacheString, metadata, false).getCommand().clone();
return QueryRewriter.rewrite(result, metadata, context);
}
Aggregations