Examples with ProjectOperator org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator used on opensource projects

Example 1 with ProjectOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator in project asterixdb by apache.

the class LangExpressionToPlanTranslator method translate.

public ILogicalPlan translate(Query expr, String outputDatasetName, ICompiledDmlStatement stmt, ILogicalOperator baseOp) throws AlgebricksException {
    MutableObject<ILogicalOperator> base = new MutableObject<>(new EmptyTupleSourceOperator());
    if (baseOp != null) {
        base = new MutableObject<>(baseOp);
    }
    Pair<ILogicalOperator, LogicalVariable> p = expr.accept(this, base);
    ArrayList<Mutable<ILogicalOperator>> globalPlanRoots = new ArrayList<>();
    ILogicalOperator topOp = p.first;
    List<LogicalVariable> liveVars = new ArrayList<>();
    VariableUtilities.getLiveVariables(topOp, liveVars);
    LogicalVariable unnestVar = liveVars.get(0);
    LogicalVariable resVar = unnestVar;
    if (outputDatasetName == null) {
        FileSplit outputFileSplit = metadataProvider.getOutputFile();
        if (outputFileSplit == null) {
            outputFileSplit = getDefaultOutputFileLocation(metadataProvider.getApplicationContext());
        }
        metadataProvider.setOutputFile(outputFileSplit);
        List<Mutable<ILogicalExpression>> writeExprList = new ArrayList<>(1);
        writeExprList.add(new MutableObject<>(new VariableReferenceExpression(resVar)));
        ResultSetSinkId rssId = new ResultSetSinkId(metadataProvider.getResultSetId());
        ResultSetDataSink sink = new ResultSetDataSink(rssId, null);
        DistributeResultOperator newTop = new DistributeResultOperator(writeExprList, sink);
        newTop.getInputs().add(new MutableObject<>(topOp));
        topOp = newTop;
        // Retrieve the Output RecordType (if any) and store it on
        // the DistributeResultOperator
        IAType outputRecordType = metadataProvider.findOutputRecordType();
        if (outputRecordType != null) {
            topOp.getAnnotations().put("output-record-type", outputRecordType);
        }
    } else {
        /**
             * add the collection-to-sequence right before the project,
             * because dataset only accept non-collection records
             */
        LogicalVariable seqVar = context.newVar();
        /**
             * This assign adds a marker function collection-to-sequence: if the input is a singleton collection, unnest
             * it; otherwise do nothing.
             */
        AssignOperator assignCollectionToSequence = new AssignOperator(seqVar, new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.COLLECTION_TO_SEQUENCE), new MutableObject<>(new VariableReferenceExpression(resVar)))));
        assignCollectionToSequence.getInputs().add(new MutableObject<>(topOp.getInputs().get(0).getValue()));
        topOp.getInputs().get(0).setValue(assignCollectionToSequence);
        ProjectOperator projectOperator = (ProjectOperator) topOp;
        projectOperator.getVariables().set(0, seqVar);
        resVar = seqVar;
        DatasetDataSource targetDatasource = validateDatasetInfo(metadataProvider, stmt.getDataverseName(), stmt.getDatasetName());
        List<Integer> keySourceIndicator = ((InternalDatasetDetails) targetDatasource.getDataset().getDatasetDetails()).getKeySourceIndicator();
        ArrayList<LogicalVariable> vars = new ArrayList<>();
        ArrayList<Mutable<ILogicalExpression>> exprs = new ArrayList<>();
        List<Mutable<ILogicalExpression>> varRefsForLoading = new ArrayList<>();
        List<List<String>> partitionKeys = targetDatasource.getDataset().getPrimaryKeys();
        int numOfPrimaryKeys = partitionKeys.size();
        for (int i = 0; i < numOfPrimaryKeys; i++) {
            if (keySourceIndicator == null || keySourceIndicator.get(i).intValue() == 0) {
                // record part
                PlanTranslationUtil.prepareVarAndExpression(partitionKeys.get(i), resVar, vars, exprs, varRefsForLoading, context);
            } else {
                // meta part
                PlanTranslationUtil.prepareMetaKeyAccessExpression(partitionKeys.get(i), unnestVar, exprs, vars, varRefsForLoading, context);
            }
        }
        AssignOperator assign = new AssignOperator(vars, exprs);
        List<String> additionalFilteringField = DatasetUtil.getFilterField(targetDatasource.getDataset());
        List<LogicalVariable> additionalFilteringVars;
        List<Mutable<ILogicalExpression>> additionalFilteringAssignExpressions;
        List<Mutable<ILogicalExpression>> additionalFilteringExpressions = null;
        AssignOperator additionalFilteringAssign = null;
        if (additionalFilteringField != null) {
            additionalFilteringVars = new ArrayList<>();
            additionalFilteringAssignExpressions = new ArrayList<>();
            additionalFilteringExpressions = new ArrayList<>();
            PlanTranslationUtil.prepareVarAndExpression(additionalFilteringField, resVar, additionalFilteringVars, additionalFilteringAssignExpressions, additionalFilteringExpressions, context);
            additionalFilteringAssign = new AssignOperator(additionalFilteringVars, additionalFilteringAssignExpressions);
            additionalFilteringAssign.getInputs().add(new MutableObject<>(topOp));
            assign.getInputs().add(new MutableObject<>(additionalFilteringAssign));
        } else {
            assign.getInputs().add(new MutableObject<>(topOp));
        }
        Mutable<ILogicalExpression> varRef = new MutableObject<>(new VariableReferenceExpression(resVar));
        ILogicalOperator leafOperator;
        switch(stmt.getKind()) {
            case Statement.Kind.INSERT:
                leafOperator = translateInsert(targetDatasource, varRef, varRefsForLoading, additionalFilteringExpressions, assign, stmt);
                break;
            case Statement.Kind.UPSERT:
                leafOperator = translateUpsert(targetDatasource, varRef, varRefsForLoading, additionalFilteringExpressions, assign, additionalFilteringField, unnestVar, topOp, exprs, resVar, additionalFilteringAssign, stmt);
                break;
            case Statement.Kind.DELETE:
                leafOperator = translateDelete(targetDatasource, varRef, varRefsForLoading, additionalFilteringExpressions, assign);
                break;
            case Statement.Kind.CONNECT_FEED:
                leafOperator = translateConnectFeed(targetDatasource, varRef, varRefsForLoading, additionalFilteringExpressions, assign);
                break;
            case Statement.Kind.SUBSCRIBE_FEED:
                leafOperator = translateSubscribeFeed((CompiledSubscribeFeedStatement) stmt, targetDatasource, unnestVar, topOp, exprs, resVar, varRefsForLoading, varRef, assign, additionalFilteringField, additionalFilteringAssign, additionalFilteringExpressions);
                break;
            default:
                throw new AlgebricksException("Unsupported statement kind " + stmt.getKind());
        }
        topOp = leafOperator;
    }
    globalPlanRoots.add(new MutableObject<>(topOp));
    ILogicalPlan plan = new ALogicalPlanImpl(globalPlanRoots);
    eliminateSharedOperatorReferenceForPlan(plan);
    return plan;
}

Example 2 with ProjectOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator in project asterixdb by apache.

the class LangExpressionToPlanTranslator method translateSubscribeFeed.

private ILogicalOperator translateSubscribeFeed(CompiledSubscribeFeedStatement sfs, DatasetDataSource targetDatasource, LogicalVariable unnestVar, ILogicalOperator topOp, ArrayList<Mutable<ILogicalExpression>> exprs, LogicalVariable resVar, List<Mutable<ILogicalExpression>> varRefsForLoading, Mutable<ILogicalExpression> varRef, ILogicalOperator assign, List<String> additionalFilteringField, AssignOperator additionalFilteringAssign, List<Mutable<ILogicalExpression>> additionalFilteringExpressions) throws AlgebricksException {
    // if the feed is a change feed (i.e, performs different operations), we need to project op variable
    InsertDeleteUpsertOperator feedModificationOp;
    AssignOperator metaAndKeysAssign;
    List<LogicalVariable> metaAndKeysVars = null;
    List<Mutable<ILogicalExpression>> metaAndKeysExprs = null;
    List<Mutable<ILogicalExpression>> metaExpSingletonList = null;
    Feed feed = metadataProvider.findFeed(sfs.getDataverseName(), sfs.getFeedName());
    boolean isChangeFeed = ExternalDataUtils.isChangeFeed(feed.getAdapterConfiguration());
    boolean isUpsertFeed = ExternalDataUtils.isUpsertFeed(feed.getAdapterConfiguration());
    ProjectOperator project = (ProjectOperator) topOp;
    if (targetDatasource.getDataset().hasMetaPart() || isChangeFeed) {
        metaAndKeysVars = new ArrayList<>();
        metaAndKeysExprs = new ArrayList<>();
        if (targetDatasource.getDataset().hasMetaPart()) {
            // add the meta function
            IFunctionInfo finfoMeta = FunctionUtil.getFunctionInfo(BuiltinFunctions.META);
            ScalarFunctionCallExpression metaFunction = new ScalarFunctionCallExpression(finfoMeta, new MutableObject<>(new VariableReferenceExpression(unnestVar)));
            // create assign for the meta part
            LogicalVariable metaVar = context.newVar();
            metaExpSingletonList = new ArrayList<>(1);
            metaExpSingletonList.add(new MutableObject<>(new VariableReferenceExpression(metaVar)));
            metaAndKeysVars.add(metaVar);
            metaAndKeysExprs.add(new MutableObject<>(metaFunction));
            project.getVariables().add(metaVar);
        }
    }
    if (isChangeFeed) {
        varRefsForLoading.clear();
        for (Mutable<ILogicalExpression> assignExpr : exprs) {
            if (assignExpr.getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
                AbstractFunctionCallExpression funcCall = (AbstractFunctionCallExpression) assignExpr.getValue();
                funcCall.substituteVar(resVar, unnestVar);
                LogicalVariable pkVar = context.newVar();
                metaAndKeysVars.add(pkVar);
                metaAndKeysExprs.add(new MutableObject<>(assignExpr.getValue()));
                project.getVariables().add(pkVar);
                varRefsForLoading.add(new MutableObject<>(new VariableReferenceExpression(pkVar)));
            }
        }
        // A change feed, we don't need the assign to access PKs
        feedModificationOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading, metaExpSingletonList, InsertDeleteUpsertOperator.Kind.UPSERT, false);
        // Create and add a new variable used for representing the original record
        feedModificationOp.setPrevRecordVar(context.newVar());
        feedModificationOp.setPrevRecordType(targetDatasource.getItemType());
        if (targetDatasource.getDataset().hasMetaPart()) {
            List<LogicalVariable> metaVars = new ArrayList<>();
            metaVars.add(context.newVar());
            feedModificationOp.setPrevAdditionalNonFilteringVars(metaVars);
            List<Object> metaTypes = new ArrayList<>();
            metaTypes.add(targetDatasource.getMetaItemType());
            feedModificationOp.setPrevAdditionalNonFilteringTypes(metaTypes);
        }
        if (additionalFilteringField != null) {
            feedModificationOp.setPrevFilterVar(context.newVar());
            feedModificationOp.setPrevFilterType(((ARecordType) targetDatasource.getItemType()).getFieldType(additionalFilteringField.get(0)));
            additionalFilteringAssign.getInputs().clear();
            additionalFilteringAssign.getInputs().add(assign.getInputs().get(0));
            feedModificationOp.getInputs().add(new MutableObject<>(additionalFilteringAssign));
        } else {
            feedModificationOp.getInputs().add(assign.getInputs().get(0));
        }
    } else {
        final InsertDeleteUpsertOperator.Kind opKind = isUpsertFeed ? InsertDeleteUpsertOperator.Kind.UPSERT : InsertDeleteUpsertOperator.Kind.INSERT;
        feedModificationOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading, metaExpSingletonList, opKind, false);
        if (isUpsertFeed) {
            feedModificationOp.setPrevRecordVar(context.newVar());
            feedModificationOp.setPrevRecordType(targetDatasource.getItemType());
        }
        feedModificationOp.getInputs().add(new MutableObject<>(assign));
    }
    if (targetDatasource.getDataset().hasMetaPart() || isChangeFeed) {
        metaAndKeysAssign = new AssignOperator(metaAndKeysVars, metaAndKeysExprs);
        metaAndKeysAssign.getInputs().add(topOp.getInputs().get(0));
        topOp.getInputs().set(0, new MutableObject<>(metaAndKeysAssign));
    }
    feedModificationOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
    ILogicalOperator leafOperator = new DelegateOperator(new CommitOperator(true));
    leafOperator.getInputs().add(new MutableObject<>(feedModificationOp));
    return leafOperator;
}

Example 3 with ProjectOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator in project asterixdb by apache.

the class LangExpressionToPlanTranslator method aggListifyForSubquery.

protected Pair<ILogicalOperator, LogicalVariable> aggListifyForSubquery(LogicalVariable var, Mutable<ILogicalOperator> opRef, boolean bProject) {
    AggregateFunctionCallExpression funAgg = BuiltinFunctions.makeAggregateFunctionExpression(BuiltinFunctions.LISTIFY, new ArrayList<>());
    funAgg.getArguments().add(new MutableObject<>(new VariableReferenceExpression(var)));
    LogicalVariable varListified = context.newSubplanOutputVar();
    AggregateOperator agg = new AggregateOperator(mkSingletonArrayList(varListified), mkSingletonArrayList(new MutableObject<>(funAgg)));
    agg.getInputs().add(opRef);
    ILogicalOperator res;
    if (bProject) {
        ProjectOperator pr = new ProjectOperator(varListified);
        pr.getInputs().add(new MutableObject<>(agg));
        res = pr;
    } else {
        res = agg;
    }
    return new Pair<>(res, varListified);
}

Example 4 with ProjectOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator in project asterixdb by apache.

the class SqlppExpressionToPlanTranslator method visit.

@Override
public Pair<ILogicalOperator, LogicalVariable> visit(Query q, Mutable<ILogicalOperator> tupSource) throws CompilationException {
    Expression queryBody = q.getBody();
    if (queryBody.getKind() == Kind.SELECT_EXPRESSION) {
        SelectExpression selectExpr = (SelectExpression) queryBody;
        if (q.isTopLevel()) {
            selectExpr.setSubquery(false);
        }
        return queryBody.accept(this, tupSource);
    } else {
        LogicalVariable var = context.newVar();
        Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = langExprToAlgExpression(queryBody, tupSource);
        AssignOperator assignOp = new AssignOperator(var, new MutableObject<ILogicalExpression>(eo.first));
        assignOp.getInputs().add(eo.second);
        ProjectOperator projectOp = new ProjectOperator(var);
        projectOp.getInputs().add(new MutableObject<ILogicalOperator>(assignOp));
        return new Pair<>(projectOp, var);
    }
}

Example 5 with ProjectOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator in project asterixdb by apache.

the class InsertProjectBeforeUnionRule method insertProjectOperator.

private void insertProjectOperator(UnionAllOperator opUnion, int branch, ArrayList<LogicalVariable> usedVariables, IOptimizationContext context) throws AlgebricksException {
    ProjectOperator projectOp = new ProjectOperator(usedVariables);
    ILogicalOperator parentOp = opUnion.getInputs().get(branch).getValue();
    projectOp.getInputs().add(new MutableObject<ILogicalOperator>(parentOp));
    opUnion.getInputs().get(branch).setValue(projectOp);
    projectOp.setPhysicalOperator(new StreamProjectPOperator());
    context.computeAndSetTypeEnvironmentForOperator(projectOp);
    context.computeAndSetTypeEnvironmentForOperator(parentOp);
}