Examples with InsertDeleteUpsertOperator org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator used on opensource projects

Example 11 with InsertDeleteUpsertOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator in project asterixdb by apache.

the class IntroduceAutogenerateIDRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    // match: commit OR distribute-result OR SINK - ... followed by:
    // [insert to internal dataset with autogenerated id] - assign - project
    // produce: insert - assign - assign* - project
    // **
    // OR [insert to internal dataset with autogenerated id] - assign - [datasource scan]
    // produce insert - assign - assign* - datasource scan
    AbstractLogicalOperator currentOp = (AbstractLogicalOperator) opRef.getValue();
    if (currentOp.getOperatorTag() == LogicalOperatorTag.DELEGATE_OPERATOR) {
        DelegateOperator dOp = (DelegateOperator) currentOp;
        if (!(dOp.getDelegate() instanceof CommitOperator)) {
            return false;
        } else if (!((CommitOperator) dOp.getDelegate()).isSink()) {
            return false;
        }
    } else if (currentOp.getOperatorTag() != LogicalOperatorTag.DISTRIBUTE_RESULT && currentOp.getOperatorTag() != LogicalOperatorTag.SINK) {
        return false;
    }
    ArrayDeque<AbstractLogicalOperator> opStack = new ArrayDeque<>();
    opStack.push(currentOp);
    while (currentOp.getInputs().size() == 1) {
        currentOp = (AbstractLogicalOperator) currentOp.getInputs().get(0).getValue();
        if (currentOp.getOperatorTag() == LogicalOperatorTag.INSERT_DELETE_UPSERT) {
            break;
        }
        opStack.push(currentOp);
    }
    if (currentOp.getOperatorTag() != LogicalOperatorTag.INSERT_DELETE_UPSERT) {
        return false;
    }
    InsertDeleteUpsertOperator insertOp = (InsertDeleteUpsertOperator) currentOp;
    if (insertOp.getOperation() != Kind.INSERT && insertOp.getOperation() != Kind.UPSERT) {
        return false;
    }
    DatasetDataSource dds = (DatasetDataSource) insertOp.getDataSource();
    boolean autogenerated = ((InternalDatasetDetails) dds.getDataset().getDatasetDetails()).isAutogenerated();
    if (!autogenerated) {
        return false;
    }
    if (((DataSource) insertOp.getDataSource()).getDatasourceType() != Type.INTERNAL_DATASET) {
        return false;
    }
    AbstractLogicalOperator parentOp = (AbstractLogicalOperator) currentOp.getInputs().get(0).getValue();
    if (parentOp.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
        return false;
    }
    AssignOperator assignOp = (AssignOperator) parentOp;
    LogicalVariable inputRecord;
    //TODO: bug here. will not work for internal datasets with filters since the pattern becomes 
    //[project-assign-assign-insert]
    AbstractLogicalOperator grandparentOp = (AbstractLogicalOperator) parentOp.getInputs().get(0).getValue();
    if (grandparentOp.getOperatorTag() == LogicalOperatorTag.PROJECT) {
        ProjectOperator projectOp = (ProjectOperator) grandparentOp;
        inputRecord = projectOp.getVariables().get(0);
    } else if (grandparentOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
        DataSourceScanOperator dssOp = (DataSourceScanOperator) grandparentOp;
        inputRecord = dssOp.getVariables().get(0);
    } else {
        return false;
    }
    List<String> pkFieldName = ((InternalDatasetDetails) dds.getDataset().getDatasetDetails()).getPrimaryKey().get(0);
    ILogicalExpression rec0 = new VariableReferenceExpression(inputRecord);
    ILogicalExpression rec1 = createPrimaryKeyRecordExpression(pkFieldName);
    ILogicalExpression mergedRec = createRecordMergeFunction(rec0, rec1);
    ILogicalExpression nonNullMergedRec = createNotNullFunction(mergedRec);
    LogicalVariable v = context.newVar();
    AssignOperator newAssign = new AssignOperator(v, new MutableObject<ILogicalExpression>(nonNullMergedRec));
    newAssign.getInputs().add(new MutableObject<ILogicalOperator>(grandparentOp));
    assignOp.getInputs().set(0, new MutableObject<ILogicalOperator>(newAssign));
    VariableUtilities.substituteVariables(assignOp, inputRecord, v, context);
    VariableUtilities.substituteVariables(insertOp, inputRecord, v, context);
    context.computeAndSetTypeEnvironmentForOperator(newAssign);
    context.computeAndSetTypeEnvironmentForOperator(assignOp);
    context.computeAndSetTypeEnvironmentForOperator(insertOp);
    ;
    for (AbstractLogicalOperator op : opStack) {
        VariableUtilities.substituteVariables(op, inputRecord, v, context);
        context.computeAndSetTypeEnvironmentForOperator(op);
    }
    return true;
}

Example 12 with InsertDeleteUpsertOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator in project asterixdb by apache.

the class LangExpressionToPlanTranslator method translateUpsert.

private ILogicalOperator translateUpsert(DatasetDataSource targetDatasource, Mutable<ILogicalExpression> varRef, List<Mutable<ILogicalExpression>> varRefsForLoading, List<Mutable<ILogicalExpression>> additionalFilteringExpressions, ILogicalOperator assign, List<String> additionalFilteringField, LogicalVariable unnestVar, ILogicalOperator topOp, List<Mutable<ILogicalExpression>> exprs, LogicalVariable resVar, AssignOperator additionalFilteringAssign, ICompiledDmlStatement stmt) throws AlgebricksException {
    if (!targetDatasource.getDataset().allow(topOp, DatasetUtil.OP_UPSERT)) {
        throw new AlgebricksException(targetDatasource.getDataset().getDatasetName() + ": upsert into dataset is not supported on Datasets with Meta records");
    }
    ProjectOperator project = (ProjectOperator) topOp;
    CompiledUpsertStatement compiledUpsert = (CompiledUpsertStatement) stmt;
    Expression returnExpression = compiledUpsert.getReturnExpression();
    InsertDeleteUpsertOperator upsertOp;
    ILogicalOperator rootOperator;
    if (targetDatasource.getDataset().hasMetaPart()) {
        if (returnExpression != null) {
            throw new AlgebricksException("Returning not allowed on datasets with Meta records");
        }
        AssignOperator metaAndKeysAssign;
        List<LogicalVariable> metaAndKeysVars;
        List<Mutable<ILogicalExpression>> metaAndKeysExprs;
        List<Mutable<ILogicalExpression>> metaExpSingletonList;
        metaAndKeysVars = new ArrayList<>();
        metaAndKeysExprs = new ArrayList<>();
        // add the meta function
        IFunctionInfo finfoMeta = FunctionUtil.getFunctionInfo(BuiltinFunctions.META);
        ScalarFunctionCallExpression metaFunction = new ScalarFunctionCallExpression(finfoMeta, new MutableObject<>(new VariableReferenceExpression(unnestVar)));
        // create assign for the meta part
        LogicalVariable metaVar = context.newVar();
        metaExpSingletonList = new ArrayList<>(1);
        metaExpSingletonList.add(new MutableObject<>(new VariableReferenceExpression(metaVar)));
        metaAndKeysVars.add(metaVar);
        metaAndKeysExprs.add(new MutableObject<>(metaFunction));
        project.getVariables().add(metaVar);
        varRefsForLoading.clear();
        for (Mutable<ILogicalExpression> assignExpr : exprs) {
            if (assignExpr.getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
                AbstractFunctionCallExpression funcCall = (AbstractFunctionCallExpression) assignExpr.getValue();
                funcCall.substituteVar(resVar, unnestVar);
                LogicalVariable pkVar = context.newVar();
                metaAndKeysVars.add(pkVar);
                metaAndKeysExprs.add(new MutableObject<>(assignExpr.getValue()));
                project.getVariables().add(pkVar);
                varRefsForLoading.add(new MutableObject<>(new VariableReferenceExpression(pkVar)));
            }
        }
        // A change feed, we don't need the assign to access PKs
        upsertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading, metaExpSingletonList, InsertDeleteUpsertOperator.Kind.UPSERT, false);
        // Create and add a new variable used for representing the original record
        upsertOp.setPrevRecordVar(context.newVar());
        upsertOp.setPrevRecordType(targetDatasource.getItemType());
        if (targetDatasource.getDataset().hasMetaPart()) {
            List<LogicalVariable> metaVars = new ArrayList<>();
            metaVars.add(context.newVar());
            upsertOp.setPrevAdditionalNonFilteringVars(metaVars);
            List<Object> metaTypes = new ArrayList<>();
            metaTypes.add(targetDatasource.getMetaItemType());
            upsertOp.setPrevAdditionalNonFilteringTypes(metaTypes);
        }
        if (additionalFilteringField != null) {
            upsertOp.setPrevFilterVar(context.newVar());
            upsertOp.setPrevFilterType(((ARecordType) targetDatasource.getItemType()).getFieldType(additionalFilteringField.get(0)));
            additionalFilteringAssign.getInputs().clear();
            additionalFilteringAssign.getInputs().add(assign.getInputs().get(0));
            upsertOp.getInputs().add(new MutableObject<>(additionalFilteringAssign));
        } else {
            upsertOp.getInputs().add(assign.getInputs().get(0));
        }
        metaAndKeysAssign = new AssignOperator(metaAndKeysVars, metaAndKeysExprs);
        metaAndKeysAssign.getInputs().add(topOp.getInputs().get(0));
        topOp.getInputs().set(0, new MutableObject<>(metaAndKeysAssign));
        upsertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
    } else {
        upsertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading, InsertDeleteUpsertOperator.Kind.UPSERT, false);
        upsertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
        upsertOp.getInputs().add(new MutableObject<>(assign));
        // Create and add a new variable used for representing the original record
        ARecordType recordType = (ARecordType) targetDatasource.getItemType();
        upsertOp.setPrevRecordVar(context.newVar());
        upsertOp.setPrevRecordType(recordType);
        if (additionalFilteringField != null) {
            upsertOp.setPrevFilterVar(context.newVar());
            upsertOp.setPrevFilterType(recordType.getFieldType(additionalFilteringField.get(0)));
        }
    }
    rootOperator = new DelegateOperator(new CommitOperator(returnExpression == null));
    rootOperator.getInputs().add(new MutableObject<>(upsertOp));
    // Compiles the return expression.
    return processReturningExpression(rootOperator, upsertOp, compiledUpsert);
}

Example 13 with InsertDeleteUpsertOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator in project asterixdb by apache.

the class LangExpressionToPlanTranslator method translateConnectFeed.

private ILogicalOperator translateConnectFeed(DatasetDataSource targetDatasource, Mutable<ILogicalExpression> varRef, List<Mutable<ILogicalExpression>> varRefsForLoading, List<Mutable<ILogicalExpression>> additionalFilteringExpressions, ILogicalOperator assign) {
    InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, false);
    insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
    insertOp.getInputs().add(new MutableObject<>(assign));
    ILogicalOperator leafOperator = new DelegateOperator(new CommitOperator(true));
    leafOperator.getInputs().add(new MutableObject<>(insertOp));
    return leafOperator;
}

Example 14 with InsertDeleteUpsertOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator in project asterixdb by apache.

the class LangExpressionToPlanTranslator method translateInsert.

private ILogicalOperator translateInsert(DatasetDataSource targetDatasource, Mutable<ILogicalExpression> varRef, List<Mutable<ILogicalExpression>> varRefsForLoading, List<Mutable<ILogicalExpression>> additionalFilteringExpressions, ILogicalOperator assign, ICompiledDmlStatement stmt) throws AlgebricksException {
    if (targetDatasource.getDataset().hasMetaPart()) {
        throw new AlgebricksException(targetDatasource.getDataset().getDatasetName() + ": insert into dataset is not supported on Datasets with Meta records");
    }
    // Adds the insert operator.
    InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, false);
    insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
    insertOp.getInputs().add(new MutableObject<>(assign));
    // Adds the commit operator.
    CompiledInsertStatement compiledInsert = (CompiledInsertStatement) stmt;
    Expression returnExpression = compiledInsert.getReturnExpression();
    ILogicalOperator rootOperator = new DelegateOperator(new CommitOperator(returnExpression == null ? true : false));
    rootOperator.getInputs().add(new MutableObject<>(insertOp));
    // Compiles the return expression.
    return processReturningExpression(rootOperator, insertOp, compiledInsert);
}

Example 15 with InsertDeleteUpsertOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator in project asterixdb by apache.

the class LangExpressionToPlanTranslator method translateDelete.

private ILogicalOperator translateDelete(DatasetDataSource targetDatasource, Mutable<ILogicalExpression> varRef, List<Mutable<ILogicalExpression>> varRefsForLoading, List<Mutable<ILogicalExpression>> additionalFilteringExpressions, ILogicalOperator assign) throws AlgebricksException {
    if (targetDatasource.getDataset().hasMetaPart()) {
        throw new AlgebricksException(targetDatasource.getDataset().getDatasetName() + ": delete from dataset is not supported on Datasets with Meta records");
    }
    InsertDeleteUpsertOperator deleteOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading, InsertDeleteUpsertOperator.Kind.DELETE, false);
    deleteOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
    deleteOp.getInputs().add(new MutableObject<>(assign));
    ILogicalOperator leafOperator = new DelegateOperator(new CommitOperator(true));
    leafOperator.getInputs().add(new MutableObject<>(deleteOp));
    return leafOperator;
}