Examples with ILogicalPlan org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan used on opensource projects

Example 76 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class FDsAndEquivClassesVisitor method visitGroupByOperator.

@Override
public Void visitGroupByOperator(GroupByOperator op, IOptimizationContext ctx) throws AlgebricksException {
    Map<LogicalVariable, EquivalenceClass> equivalenceClasses = new HashMap<LogicalVariable, EquivalenceClass>();
    List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
    ctx.putEquivalenceClassMap(op, equivalenceClasses);
    ctx.putFDList(op, functionalDependencies);
    List<FunctionalDependency> inheritedFDs = new ArrayList<FunctionalDependency>();
    for (ILogicalPlan p : op.getNestedPlans()) {
        for (Mutable<ILogicalOperator> r : p.getRoots()) {
            ILogicalOperator op2 = r.getValue();
            equivalenceClasses.putAll(getOrComputeEqClasses(op2, ctx));
            inheritedFDs.addAll(getOrComputeFDs(op2, ctx));
        }
    }
    ILogicalOperator op0 = op.getInputs().get(0).getValue();
    inheritedFDs.addAll(getOrComputeFDs(op0, ctx));
    Map<LogicalVariable, EquivalenceClass> inheritedEcs = getOrComputeEqClasses(op0, ctx);
    for (FunctionalDependency inherited : inheritedFDs) {
        boolean isCoveredByGbyOrDecorVars = true;
        List<LogicalVariable> newHead = new ArrayList<LogicalVariable>(inherited.getHead().size());
        for (LogicalVariable v : inherited.getHead()) {
            LogicalVariable vnew = getNewGbyVar(op, v);
            if (vnew == null) {
                vnew = getNewDecorVar(op, v);
                if (vnew == null) {
                    isCoveredByGbyOrDecorVars = false;
                }
                break;
            }
            newHead.add(vnew);
        }
        if (isCoveredByGbyOrDecorVars) {
            List<LogicalVariable> newTail = new ArrayList<LogicalVariable>();
            for (LogicalVariable v2 : inherited.getTail()) {
                LogicalVariable v3 = getNewGbyVar(op, v2);
                if (v3 != null) {
                    newTail.add(v3);
                }
            }
            if (!newTail.isEmpty()) {
                FunctionalDependency newFd = new FunctionalDependency(newHead, newTail);
                functionalDependencies.add(newFd);
            }
        }
    }
    List<LogicalVariable> premiseGby = new LinkedList<LogicalVariable>();
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gByList = op.getGroupByList();
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
        premiseGby.add(p.first);
    }
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList = op.getDecorList();
    LinkedList<LogicalVariable> conclDecor = new LinkedList<LogicalVariable>();
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
        conclDecor.add(GroupByOperator.getDecorVariable(p));
    }
    if (!conclDecor.isEmpty()) {
        functionalDependencies.add(new FunctionalDependency(premiseGby, conclDecor));
    }
    Set<LogicalVariable> gbySet = new HashSet<LogicalVariable>();
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
        ILogicalExpression expr = p.second.getValue();
        if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
            VariableReferenceExpression v = (VariableReferenceExpression) expr;
            gbySet.add(v.getVariableReference());
        }
    }
    LocalGroupingProperty lgp = new LocalGroupingProperty(gbySet);
    ILocalStructuralProperty normalizedLgp = lgp.normalize(inheritedEcs, inheritedFDs);
    Set<LogicalVariable> normSet = new ListSet<>();
    normalizedLgp.getColumns(normSet);
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> newGbyList = new ArrayList<>();
    boolean changed = false;
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
        ILogicalExpression expr = p.second.getValue();
        if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
            VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
            LogicalVariable v2 = varRef.getVariableReference();
            EquivalenceClass ec2 = inheritedEcs.get(v2);
            LogicalVariable v3;
            if (ec2 != null && !ec2.representativeIsConst()) {
                v3 = ec2.getVariableRepresentative();
            } else {
                v3 = v2;
            }
            if (normSet.contains(v3)) {
                newGbyList.add(p);
            } else {
                changed = true;
                decorList.add(p);
            }
        } else {
            newGbyList.add(p);
        }
    }
    if (changed) {
        AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Group-by list changed from " + GroupByOperator.veListToString(gByList) + " to " + GroupByOperator.veListToString(newGbyList) + ".\n");
    }
    gByList.clear();
    gByList.addAll(newGbyList);
    return null;
}

Example 77 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class FuzzyJoinRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    // current opperator is join
    if (op.getOperatorTag() != LogicalOperatorTag.INNERJOIN && op.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
        return false;
    }
    // Find GET_ITEM function.
    AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) op;
    Mutable<ILogicalExpression> expRef = joinOp.getCondition();
    Mutable<ILogicalExpression> getItemExprRef = getSimilarityExpression(expRef);
    if (getItemExprRef == null) {
        return false;
    }
    // Check if the GET_ITEM function is on one of the supported similarity-check functions.
    AbstractFunctionCallExpression getItemFuncExpr = (AbstractFunctionCallExpression) getItemExprRef.getValue();
    Mutable<ILogicalExpression> argRef = getItemFuncExpr.getArguments().get(0);
    AbstractFunctionCallExpression simFuncExpr = (AbstractFunctionCallExpression) argRef.getValue();
    if (!simFuncs.contains(simFuncExpr.getFunctionIdentifier())) {
        return false;
    }
    // Skip this rule based on annotations.
    if (simFuncExpr.getAnnotations().containsKey(IndexedNLJoinExpressionAnnotation.INSTANCE)) {
        return false;
    }
    List<Mutable<ILogicalOperator>> inputOps = joinOp.getInputs();
    ILogicalOperator leftInputOp = inputOps.get(0).getValue();
    ILogicalOperator rightInputOp = inputOps.get(1).getValue();
    List<Mutable<ILogicalExpression>> inputExps = simFuncExpr.getArguments();
    ILogicalExpression inputExp0 = inputExps.get(0).getValue();
    ILogicalExpression inputExp1 = inputExps.get(1).getValue();
    // left and right expressions are variables
    if (inputExp0.getExpressionTag() != LogicalExpressionTag.VARIABLE || inputExp1.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
        return false;
    }
    LogicalVariable inputVar0 = ((VariableReferenceExpression) inputExp0).getVariableReference();
    LogicalVariable inputVar1 = ((VariableReferenceExpression) inputExp1).getVariableReference();
    LogicalVariable leftInputVar;
    LogicalVariable rightInputVar;
    liveVars.clear();
    VariableUtilities.getLiveVariables(leftInputOp, liveVars);
    if (liveVars.contains(inputVar0)) {
        leftInputVar = inputVar0;
        rightInputVar = inputVar1;
    } else {
        leftInputVar = inputVar1;
        rightInputVar = inputVar0;
    }
    List<LogicalVariable> leftInputPKs = context.findPrimaryKey(leftInputVar);
    List<LogicalVariable> rightInputPKs = context.findPrimaryKey(rightInputVar);
    // Bail if primary keys could not be inferred.
    if (leftInputPKs == null || rightInputPKs == null) {
        return false;
    }
    // primary key has only one variable
    if (leftInputPKs.size() != 1 || rightInputPKs.size() != 1) {
        return false;
    }
    IAType leftType = (IAType) context.getOutputTypeEnvironment(leftInputOp).getVarType(leftInputVar);
    IAType rightType = (IAType) context.getOutputTypeEnvironment(rightInputOp).getVarType(rightInputVar);
    // left-hand side and right-hand side of "~=" has the same type
    IAType left2 = TypeComputeUtils.getActualType(leftType);
    IAType right2 = TypeComputeUtils.getActualType(rightType);
    if (!left2.deepEqual(right2)) {
        return false;
    }
    //
    // -- - FIRE - --
    //
    MetadataProvider metadataProvider = ((MetadataProvider) context.getMetadataProvider());
    FunctionIdentifier funcId = FuzzyUtils.getTokenizer(leftType.getTypeTag());
    String tokenizer;
    if (funcId == null) {
        tokenizer = "";
    } else {
        tokenizer = funcId.getName();
    }
    float simThreshold = FuzzyUtils.getSimThreshold(metadataProvider);
    String simFunction = FuzzyUtils.getSimFunction(metadataProvider);
    // finalize AQL+ query
    String prepareJoin;
    switch(joinOp.getJoinKind()) {
        case INNER:
            {
                prepareJoin = "join" + AQLPLUS;
                break;
            }
        case LEFT_OUTER:
            {
                // other sort of bug.
                return false;
            // prepareJoin = "loj" + AQLPLUS;
            // break;
            }
        default:
            {
                throw new IllegalStateException();
            }
    }
    String aqlPlus = String.format(Locale.US, prepareJoin, tokenizer, tokenizer, simFunction, simThreshold, tokenizer, tokenizer, simFunction, simThreshold, simFunction, simThreshold, simThreshold);
    LogicalVariable leftPKVar = leftInputPKs.get(0);
    LogicalVariable rightPKVar = rightInputPKs.get(0);
    Counter counter = new Counter(context.getVarCounter());
    AQLPlusParser parser = new AQLPlusParser(new StringReader(aqlPlus));
    parser.initScope();
    parser.setVarCounter(counter);
    List<Clause> clauses;
    try {
        clauses = parser.Clauses();
    } catch (ParseException e) {
        throw new AlgebricksException(e);
    }
    // The translator will compile metadata internally. Run this compilation
    // under the same transaction id as the "outer" compilation.
    AqlPlusExpressionToPlanTranslator translator = new AqlPlusExpressionToPlanTranslator(metadataProvider, counter);
    context.setVarCounter(counter.get());
    LogicalOperatorDeepCopyWithNewVariablesVisitor deepCopyVisitor = new LogicalOperatorDeepCopyWithNewVariablesVisitor(context, context);
    translator.addOperatorToMetaScope(new Identifier("#LEFT"), leftInputOp);
    translator.addVariableToMetaScope(new Identifier("$$LEFT"), leftInputVar);
    translator.addVariableToMetaScope(new Identifier("$$LEFTPK"), leftPKVar);
    translator.addOperatorToMetaScope(new Identifier("#RIGHT"), rightInputOp);
    translator.addVariableToMetaScope(new Identifier("$$RIGHT"), rightInputVar);
    translator.addVariableToMetaScope(new Identifier("$$RIGHTPK"), rightPKVar);
    translator.addOperatorToMetaScope(new Identifier("#LEFT_1"), deepCopyVisitor.deepCopy(leftInputOp));
    translator.addVariableToMetaScope(new Identifier("$$LEFT_1"), deepCopyVisitor.varCopy(leftInputVar));
    translator.addVariableToMetaScope(new Identifier("$$LEFTPK_1"), deepCopyVisitor.varCopy(leftPKVar));
    deepCopyVisitor.updatePrimaryKeys(context);
    deepCopyVisitor.reset();
    // translator.addOperatorToMetaScope(new Identifier("#LEFT_2"),
    // deepCopyVisitor.deepCopy(leftInputOp, null));
    // translator.addVariableToMetaScope(new Identifier("$$LEFT_2"),
    // deepCopyVisitor.varCopy(leftInputVar));
    // translator.addVariableToMetaScope(new Identifier("$$LEFTPK_2"),
    // deepCopyVisitor.varCopy(leftPKVar));
    // deepCopyVisitor.updatePrimaryKeys(context);
    // deepCopyVisitor.reset();
    //
    // translator.addOperatorToMetaScope(new Identifier("#LEFT_3"),
    // deepCopyVisitor.deepCopy(leftInputOp, null));
    // translator.addVariableToMetaScope(new Identifier("$$LEFT_3"),
    // deepCopyVisitor.varCopy(leftInputVar));
    // translator.addVariableToMetaScope(new Identifier("$$LEFTPK_3"),
    // deepCopyVisitor.varCopy(leftPKVar));
    // deepCopyVisitor.updatePrimaryKeys(context);
    // deepCopyVisitor.reset();
    translator.addOperatorToMetaScope(new Identifier("#RIGHT_1"), deepCopyVisitor.deepCopy(rightInputOp));
    translator.addVariableToMetaScope(new Identifier("$$RIGHT_1"), deepCopyVisitor.varCopy(rightInputVar));
    translator.addVariableToMetaScope(new Identifier("$$RIGHTPK_1"), deepCopyVisitor.varCopy(rightPKVar));
    deepCopyVisitor.updatePrimaryKeys(context);
    deepCopyVisitor.reset();
    // TODO pick side to run Stage 1, currently always picks RIGHT side
    translator.addOperatorToMetaScope(new Identifier("#RIGHT_2"), deepCopyVisitor.deepCopy(rightInputOp));
    translator.addVariableToMetaScope(new Identifier("$$RIGHT_2"), deepCopyVisitor.varCopy(rightInputVar));
    translator.addVariableToMetaScope(new Identifier("$$RIGHTPK_2"), deepCopyVisitor.varCopy(rightPKVar));
    deepCopyVisitor.updatePrimaryKeys(context);
    deepCopyVisitor.reset();
    translator.addOperatorToMetaScope(new Identifier("#RIGHT_3"), deepCopyVisitor.deepCopy(rightInputOp));
    translator.addVariableToMetaScope(new Identifier("$$RIGHT_3"), deepCopyVisitor.varCopy(rightInputVar));
    translator.addVariableToMetaScope(new Identifier("$$RIGHTPK_3"), deepCopyVisitor.varCopy(rightPKVar));
    deepCopyVisitor.updatePrimaryKeys(context);
    deepCopyVisitor.reset();
    ILogicalPlan plan;
    try {
        plan = translator.translate(clauses);
    } catch (AsterixException e) {
        throw new AlgebricksException(e);
    }
    context.setVarCounter(counter.get());
    ILogicalOperator outputOp = plan.getRoots().get(0).getValue();
    SelectOperator extraSelect = null;
    if (getItemExprRef != expRef) {
        // more than one join condition
        getItemExprRef.setValue(ConstantExpression.TRUE);
        switch(joinOp.getJoinKind()) {
            case INNER:
                {
                    extraSelect = new SelectOperator(expRef, false, null);
                    extraSelect.getInputs().add(new MutableObject<ILogicalOperator>(outputOp));
                    outputOp = extraSelect;
                    break;
                }
            case LEFT_OUTER:
                {
                    if (((AbstractLogicalOperator) outputOp).getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
                        throw new IllegalStateException();
                    }
                    LeftOuterJoinOperator topJoin = (LeftOuterJoinOperator) outputOp;
                    topJoin.getCondition().setValue(expRef.getValue());
                    break;
                }
            default:
                {
                    throw new IllegalStateException();
                }
        }
    }
    opRef.setValue(outputOp);
    OperatorPropertiesUtil.typeOpRec(opRef, context);
    return true;
}

Example 78 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class AbstractPhysicalOperator method compileSubplans.

protected AlgebricksPipeline[] compileSubplans(IOperatorSchema outerPlanSchema, AbstractOperatorWithNestedPlans npOp, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
    AlgebricksPipeline[] subplans = new AlgebricksPipeline[npOp.getNestedPlans().size()];
    PlanCompiler pc = new PlanCompiler(context);
    int i = 0;
    for (ILogicalPlan p : npOp.getNestedPlans()) {
        subplans[i++] = buildPipelineWithProjection(p, outerPlanSchema, npOp, opSchema, pc);
    }
    return subplans;
}

Example 79 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class OperatorDeepCopyVisitor method visitSubplanOperator.

@Override
public ILogicalOperator visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
    ArrayList<ILogicalPlan> newSubplans = new ArrayList<>();
    SubplanOperator subplanOp = new SubplanOperator(newSubplans);
    for (ILogicalPlan plan : op.getNestedPlans()) {
        newSubplans.add(OperatorManipulationUtil.deepCopy(plan, subplanOp));
    }
    return subplanOp;
}

Example 80 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class OperatorDeepCopyVisitor method visitGroupByOperator.

@Override
public ILogicalOperator visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList = new ArrayList<>();
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decoList = new ArrayList<>();
    ArrayList<ILogicalPlan> newSubplans = new ArrayList<>();
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : op.getGroupByList()) {
        groupByList.add(new Pair<>(pair.first, deepCopyExpressionRef(pair.second)));
    }
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : op.getDecorList()) {
        decoList.add(new Pair<>(pair.first, deepCopyExpressionRef(pair.second)));
    }
    GroupByOperator gbyOp = new GroupByOperator(groupByList, decoList, newSubplans, op.isGroupAll());
    for (ILogicalPlan plan : op.getNestedPlans()) {
        newSubplans.add(OperatorManipulationUtil.deepCopy(plan, gbyOp));
    }
    return gbyOp;
}