Examples with ILogicalPlan org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan used on opensource projects

Example 1 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class LangExpressionToPlanTranslator method translate.

public ILogicalPlan translate(Query expr, String outputDatasetName, ICompiledDmlStatement stmt, ILogicalOperator baseOp) throws AlgebricksException {
    MutableObject<ILogicalOperator> base = new MutableObject<>(new EmptyTupleSourceOperator());
    if (baseOp != null) {
        base = new MutableObject<>(baseOp);
    }
    Pair<ILogicalOperator, LogicalVariable> p = expr.accept(this, base);
    ArrayList<Mutable<ILogicalOperator>> globalPlanRoots = new ArrayList<>();
    ILogicalOperator topOp = p.first;
    List<LogicalVariable> liveVars = new ArrayList<>();
    VariableUtilities.getLiveVariables(topOp, liveVars);
    LogicalVariable unnestVar = liveVars.get(0);
    LogicalVariable resVar = unnestVar;
    if (outputDatasetName == null) {
        FileSplit outputFileSplit = metadataProvider.getOutputFile();
        if (outputFileSplit == null) {
            outputFileSplit = getDefaultOutputFileLocation(metadataProvider.getApplicationContext());
        }
        metadataProvider.setOutputFile(outputFileSplit);
        List<Mutable<ILogicalExpression>> writeExprList = new ArrayList<>(1);
        writeExprList.add(new MutableObject<>(new VariableReferenceExpression(resVar)));
        ResultSetSinkId rssId = new ResultSetSinkId(metadataProvider.getResultSetId());
        ResultSetDataSink sink = new ResultSetDataSink(rssId, null);
        DistributeResultOperator newTop = new DistributeResultOperator(writeExprList, sink);
        newTop.getInputs().add(new MutableObject<>(topOp));
        topOp = newTop;
        // Retrieve the Output RecordType (if any) and store it on
        // the DistributeResultOperator
        IAType outputRecordType = metadataProvider.findOutputRecordType();
        if (outputRecordType != null) {
            topOp.getAnnotations().put("output-record-type", outputRecordType);
        }
    } else {
        /**
             * add the collection-to-sequence right before the project,
             * because dataset only accept non-collection records
             */
        LogicalVariable seqVar = context.newVar();
        /**
             * This assign adds a marker function collection-to-sequence: if the input is a singleton collection, unnest
             * it; otherwise do nothing.
             */
        AssignOperator assignCollectionToSequence = new AssignOperator(seqVar, new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.COLLECTION_TO_SEQUENCE), new MutableObject<>(new VariableReferenceExpression(resVar)))));
        assignCollectionToSequence.getInputs().add(new MutableObject<>(topOp.getInputs().get(0).getValue()));
        topOp.getInputs().get(0).setValue(assignCollectionToSequence);
        ProjectOperator projectOperator = (ProjectOperator) topOp;
        projectOperator.getVariables().set(0, seqVar);
        resVar = seqVar;
        DatasetDataSource targetDatasource = validateDatasetInfo(metadataProvider, stmt.getDataverseName(), stmt.getDatasetName());
        List<Integer> keySourceIndicator = ((InternalDatasetDetails) targetDatasource.getDataset().getDatasetDetails()).getKeySourceIndicator();
        ArrayList<LogicalVariable> vars = new ArrayList<>();
        ArrayList<Mutable<ILogicalExpression>> exprs = new ArrayList<>();
        List<Mutable<ILogicalExpression>> varRefsForLoading = new ArrayList<>();
        List<List<String>> partitionKeys = targetDatasource.getDataset().getPrimaryKeys();
        int numOfPrimaryKeys = partitionKeys.size();
        for (int i = 0; i < numOfPrimaryKeys; i++) {
            if (keySourceIndicator == null || keySourceIndicator.get(i).intValue() == 0) {
                // record part
                PlanTranslationUtil.prepareVarAndExpression(partitionKeys.get(i), resVar, vars, exprs, varRefsForLoading, context);
            } else {
                // meta part
                PlanTranslationUtil.prepareMetaKeyAccessExpression(partitionKeys.get(i), unnestVar, exprs, vars, varRefsForLoading, context);
            }
        }
        AssignOperator assign = new AssignOperator(vars, exprs);
        List<String> additionalFilteringField = DatasetUtil.getFilterField(targetDatasource.getDataset());
        List<LogicalVariable> additionalFilteringVars;
        List<Mutable<ILogicalExpression>> additionalFilteringAssignExpressions;
        List<Mutable<ILogicalExpression>> additionalFilteringExpressions = null;
        AssignOperator additionalFilteringAssign = null;
        if (additionalFilteringField != null) {
            additionalFilteringVars = new ArrayList<>();
            additionalFilteringAssignExpressions = new ArrayList<>();
            additionalFilteringExpressions = new ArrayList<>();
            PlanTranslationUtil.prepareVarAndExpression(additionalFilteringField, resVar, additionalFilteringVars, additionalFilteringAssignExpressions, additionalFilteringExpressions, context);
            additionalFilteringAssign = new AssignOperator(additionalFilteringVars, additionalFilteringAssignExpressions);
            additionalFilteringAssign.getInputs().add(new MutableObject<>(topOp));
            assign.getInputs().add(new MutableObject<>(additionalFilteringAssign));
        } else {
            assign.getInputs().add(new MutableObject<>(topOp));
        }
        Mutable<ILogicalExpression> varRef = new MutableObject<>(new VariableReferenceExpression(resVar));
        ILogicalOperator leafOperator;
        switch(stmt.getKind()) {
            case Statement.Kind.INSERT:
                leafOperator = translateInsert(targetDatasource, varRef, varRefsForLoading, additionalFilteringExpressions, assign, stmt);
                break;
            case Statement.Kind.UPSERT:
                leafOperator = translateUpsert(targetDatasource, varRef, varRefsForLoading, additionalFilteringExpressions, assign, additionalFilteringField, unnestVar, topOp, exprs, resVar, additionalFilteringAssign, stmt);
                break;
            case Statement.Kind.DELETE:
                leafOperator = translateDelete(targetDatasource, varRef, varRefsForLoading, additionalFilteringExpressions, assign);
                break;
            case Statement.Kind.CONNECT_FEED:
                leafOperator = translateConnectFeed(targetDatasource, varRef, varRefsForLoading, additionalFilteringExpressions, assign);
                break;
            case Statement.Kind.SUBSCRIBE_FEED:
                leafOperator = translateSubscribeFeed((CompiledSubscribeFeedStatement) stmt, targetDatasource, unnestVar, topOp, exprs, resVar, varRefsForLoading, varRef, assign, additionalFilteringField, additionalFilteringAssign, additionalFilteringExpressions);
                break;
            default:
                throw new AlgebricksException("Unsupported statement kind " + stmt.getKind());
        }
        topOp = leafOperator;
    }
    globalPlanRoots.add(new MutableObject<>(topOp));
    ILogicalPlan plan = new ALogicalPlanImpl(globalPlanRoots);
    eliminateSharedOperatorReferenceForPlan(plan);
    return plan;
}

Example 2 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class LangExpressionToPlanTranslator method constructSubplanOperatorForBranch.

/**
     * Constructs a subplan operator for a branch in a if-else (or case) expression.
     *
     * @param inputOp,
     *            the input operator.
     * @param selectExpr,
     *            the expression to select tuples that are processed by this branch.
     * @param branchExpression,
     *            the expression to be evaluated in this branch.
     * @return a pair of the constructed subplan operator and the output variable for the branch.
     * @throws CompilationException
     */
protected Pair<ILogicalOperator, LogicalVariable> constructSubplanOperatorForBranch(ILogicalOperator inputOp, Mutable<ILogicalExpression> selectExpr, Expression branchExpression) throws CompilationException {
    context.enterSubplan();
    SubplanOperator subplanOp = new SubplanOperator();
    subplanOp.getInputs().add(new MutableObject<>(inputOp));
    Mutable<ILogicalOperator> nestedSource = new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<>(subplanOp)));
    SelectOperator select = new SelectOperator(selectExpr, false, null);
    // The select operator cannot be moved up and down, otherwise it will cause typing issues (ASTERIXDB-1203).
    OperatorPropertiesUtil.markMovable(select, false);
    select.getInputs().add(nestedSource);
    Pair<ILogicalOperator, LogicalVariable> pBranch = branchExpression.accept(this, new MutableObject<>(select));
    LogicalVariable branchVar = context.newVar();
    AggregateOperator aggOp = new AggregateOperator(Collections.singletonList(branchVar), Collections.singletonList(new MutableObject<>(new AggregateFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.LISTIFY), false, Collections.singletonList(new MutableObject<>(new VariableReferenceExpression(pBranch.second)))))));
    aggOp.getInputs().add(new MutableObject<>(pBranch.first));
    ILogicalPlan planForBranch = new ALogicalPlanImpl(new MutableObject<>(aggOp));
    subplanOp.getNestedPlans().add(planForBranch);
    context.exitSubplan();
    return new Pair<>(subplanOp, branchVar);
}

Example 3 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class AqlPlusExpressionToPlanTranslator method translate.

public ILogicalPlan translate(List<Clause> clauses) throws AlgebricksException, CompilationException {
    if (clauses == null) {
        return null;
    }
    Mutable<ILogicalOperator> opRef = new MutableObject<ILogicalOperator>(new EmptyTupleSourceOperator());
    Pair<ILogicalOperator, LogicalVariable> p = null;
    for (Clause c : clauses) {
        p = c.accept(this, opRef);
        opRef = new MutableObject<ILogicalOperator>(p.first);
    }
    ArrayList<Mutable<ILogicalOperator>> globalPlanRoots = new ArrayList<Mutable<ILogicalOperator>>();
    ILogicalOperator topOp = p.first;
    globalPlanRoots.add(new MutableObject<ILogicalOperator>(topOp));
    ILogicalPlan plan = new ALogicalPlanImpl(globalPlanRoots);
    return plan;
}

Example 4 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class EnforceStructuralPropertiesRule method physOptimizeOp.

private boolean physOptimizeOp(Mutable<ILogicalOperator> opRef, IPhysicalPropertiesVector required, boolean nestedPlan, IOptimizationContext context) throws AlgebricksException {
    boolean changed = false;
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    optimizeUsingConstraintsAndEquivClasses(op);
    PhysicalRequirements pr = op.getRequiredPhysicalPropertiesForChildren(required, context);
    IPhysicalPropertiesVector[] reqdProperties = null;
    if (pr != null) {
        reqdProperties = pr.getRequiredProperties();
    }
    boolean opIsRedundantSort = false;
    // compute properties and figure out the domain
    INodeDomain childrenDomain = null;
    {
        int j = 0;
        for (Mutable<ILogicalOperator> childRef : op.getInputs()) {
            AbstractLogicalOperator child = (AbstractLogicalOperator) childRef.getValue();
            // recursive call
            if (physOptimizeOp(childRef, reqdProperties[j], nestedPlan, context)) {
                changed = true;
            }
            child.computeDeliveredPhysicalProperties(context);
            IPhysicalPropertiesVector delivered = child.getDeliveredPhysicalProperties();
            if (childrenDomain == null) {
                childrenDomain = delivered.getPartitioningProperty().getNodeDomain();
            } else {
                INodeDomain dom2 = delivered.getPartitioningProperty().getNodeDomain();
                if (!childrenDomain.sameAs(dom2)) {
                    childrenDomain = context.getComputationNodeDomain();
                }
            }
            j++;
        }
    }
    if (reqdProperties != null) {
        for (int k = 0; k < reqdProperties.length; k++) {
            IPhysicalPropertiesVector pv = reqdProperties[k];
            IPartitioningProperty pp = pv.getPartitioningProperty();
            if (pp != null && pp.getNodeDomain() == null) {
                pp.setNodeDomain(childrenDomain);
            }
        }
    }
    // The child index of the child operator to optimize first.
    int startChildIndex = getStartChildIndex(op, pr, nestedPlan, context);
    IPartitioningProperty firstDeliveredPartitioning = null;
    // Enforce data properties in a top-down manner.
    for (int j = 0; j < op.getInputs().size(); j++) {
        // Starts from a partitioning-compatible child if any to loop over all children.
        int childIndex = (j + startChildIndex) % op.getInputs().size();
        IPhysicalPropertiesVector requiredProperty = reqdProperties[childIndex];
        AbstractLogicalOperator child = (AbstractLogicalOperator) op.getInputs().get(childIndex).getValue();
        IPhysicalPropertiesVector delivered = child.getDeliveredPhysicalProperties();
        AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Properties delivered by " + child.getPhysicalOperator() + ": " + delivered + "\n");
        IPartitioningRequirementsCoordinator prc = pr.getPartitioningCoordinator();
        // Coordinates requirements by looking at the firstDeliveredPartitioning.
        Pair<Boolean, IPartitioningProperty> pbpp = prc.coordinateRequirements(requiredProperty.getPartitioningProperty(), firstDeliveredPartitioning, op, context);
        boolean mayExpandPartitioningProperties = pbpp.first;
        IPhysicalPropertiesVector rqd = new StructuralPropertiesVector(pbpp.second, requiredProperty.getLocalProperties());
        AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Required properties for " + child.getPhysicalOperator() + ": " + rqd + "\n");
        // The partitioning property of reqdProperties[childIndex] could be updated here because
        // rqd.getPartitioningProperty() is the same object instance as requiredProperty.getPartitioningProperty().
        IPhysicalPropertiesVector diff = delivered.getUnsatisfiedPropertiesFrom(rqd, mayExpandPartitioningProperties, context.getEquivalenceClassMap(child), context.getFDList(child));
        if (isRedundantSort(opRef, delivered, diff, context)) {
            opIsRedundantSort = true;
        }
        if (diff != null) {
            changed = true;
            addEnforcers(op, childIndex, diff, rqd, delivered, childrenDomain, nestedPlan, context);
            AbstractLogicalOperator newChild = (AbstractLogicalOperator) op.getInputs().get(childIndex).getValue();
            if (newChild != child) {
                delivered = newChild.getDeliveredPhysicalProperties();
                IPhysicalPropertiesVector newDiff = newPropertiesDiff(newChild, rqd, mayExpandPartitioningProperties, context);
                AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> New properties diff: " + newDiff + "\n");
                if (isRedundantSort(opRef, delivered, newDiff, context)) {
                    opIsRedundantSort = true;
                    break;
                }
            }
        }
        if (firstDeliveredPartitioning == null) {
            firstDeliveredPartitioning = delivered.getPartitioningProperty();
        }
    }
    if (op.hasNestedPlans()) {
        AbstractOperatorWithNestedPlans nested = (AbstractOperatorWithNestedPlans) op;
        for (ILogicalPlan p : nested.getNestedPlans()) {
            if (physOptimizePlan(p, required, true, context)) {
                changed = true;
            }
        }
    }
    if (opIsRedundantSort) {
        if (AlgebricksConfig.DEBUG) {
            AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Removing redundant SORT operator " + op.getPhysicalOperator() + "\n");
            printOp(op);
        }
        changed = true;
        AbstractLogicalOperator nextOp = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
        if (nextOp.getOperatorTag() == LogicalOperatorTag.PROJECT) {
            nextOp = (AbstractLogicalOperator) nextOp.getInputs().get(0).getValue();
        }
        opRef.setValue(nextOp);
        // Now, transfer annotations from the original sort op. to this one.
        AbstractLogicalOperator transferTo = nextOp;
        if (transferTo.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
            // remove duplicate exchange operator
            transferTo = (AbstractLogicalOperator) transferTo.getInputs().get(0).getValue();
        }
        transferTo.getAnnotations().putAll(op.getAnnotations());
        physOptimizeOp(opRef, required, nestedPlan, context);
    }
    return changed;
}

Example 5 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class AbstractDecorrelationRule method buildVarExprList.

protected void buildVarExprList(Collection<LogicalVariable> vars, IOptimizationContext context, GroupByOperator g, List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> outVeList) throws AlgebricksException {
    for (LogicalVariable ov : vars) {
        LogicalVariable newVar = context.newVar();
        ILogicalExpression varExpr = new VariableReferenceExpression(newVar);
        outVeList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(ov, new MutableObject<ILogicalExpression>(varExpr)));
        for (ILogicalPlan p : g.getNestedPlans()) {
            for (Mutable<ILogicalOperator> r : p.getRoots()) {
                OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), ov, newVar, true, context);
            }
        }
    }
}