Examples with CommitOperator org.apache.asterix.algebra.operators.CommitOperator used on opensource projects

Example 1 with CommitOperator

use of org.apache.asterix.algebra.operators.CommitOperator in project asterixdb by apache.

the class IntroduceSecondaryIndexInsertDeleteRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
    if (op0.getOperatorTag() != LogicalOperatorTag.DELEGATE_OPERATOR && op0.getOperatorTag() != LogicalOperatorTag.SINK) {
        return false;
    }
    if (op0.getOperatorTag() == LogicalOperatorTag.DELEGATE_OPERATOR) {
        DelegateOperator eOp = (DelegateOperator) op0;
        if (!(eOp.getDelegate() instanceof CommitOperator)) {
            return false;
        }
    }
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) op0.getInputs().get(0).getValue();
    if (op1.getOperatorTag() != LogicalOperatorTag.INSERT_DELETE_UPSERT) {
        return false;
    }
    /** find the record variable */
    InsertDeleteUpsertOperator primaryIndexModificationOp = (InsertDeleteUpsertOperator) op0.getInputs().get(0).getValue();
    boolean isBulkload = primaryIndexModificationOp.isBulkload();
    ILogicalExpression newRecordExpr = primaryIndexModificationOp.getPayloadExpression().getValue();
    List<Mutable<ILogicalExpression>> newMetaExprs = primaryIndexModificationOp.getAdditionalNonFilteringExpressions();
    LogicalVariable newRecordVar;
    LogicalVariable newMetaVar = null;
    /**
         * inputOp is the assign operator which extracts primary keys from the input
         * variables (record or meta)
         */
    AbstractLogicalOperator inputOp = (AbstractLogicalOperator) primaryIndexModificationOp.getInputs().get(0).getValue();
    newRecordVar = getRecordVar(context, inputOp, newRecordExpr, 0);
    if (newMetaExprs != null && !newMetaExprs.isEmpty()) {
        if (newMetaExprs.size() > 1) {
            throw new AlgebricksException("Number of meta records can't be more than 1. Number of meta records found = " + newMetaExprs.size());
        }
        newMetaVar = getRecordVar(context, inputOp, newMetaExprs.get(0).getValue(), 1);
    }
    /*
         * At this point, we have the record variable and the insert/delete/upsert operator
         * Note: We have two operators:
         * 1. An InsertDeleteOperator (primary)
         * 2. An IndexInsertDeleteOperator (secondary)
         * The current primaryIndexModificationOp is of the first type
         */
    DataSource datasetSource = (DataSource) primaryIndexModificationOp.getDataSource();
    MetadataProvider mp = (MetadataProvider) context.getMetadataProvider();
    String dataverseName = datasetSource.getId().getDataverseName();
    String datasetName = datasetSource.getId().getDatasourceName();
    Dataset dataset = mp.findDataset(dataverseName, datasetName);
    if (dataset == null) {
        throw new AlgebricksException("Unknown dataset " + datasetName + " in dataverse " + dataverseName);
    }
    if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
        return false;
    }
    // Create operators for secondary index insert / delete.
    String itemTypeName = dataset.getItemTypeName();
    IAType itemType = mp.findType(dataset.getItemTypeDataverseName(), itemTypeName);
    if (itemType.getTypeTag() != ATypeTag.OBJECT) {
        throw new AlgebricksException("Only record types can be indexed.");
    }
    ARecordType recType = (ARecordType) itemType;
    // meta type
    ARecordType metaType = null;
    if (dataset.hasMetaPart()) {
        metaType = (ARecordType) mp.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
    }
    List<Index> indexes = mp.getDatasetIndexes(dataset.getDataverseName(), dataset.getDatasetName());
    // Set the top operator pointer to the primary IndexInsertDeleteOperator
    ILogicalOperator currentTop = primaryIndexModificationOp;
    boolean hasSecondaryIndex = false;
    // Put an n-gram or a keyword index in the later stage of index-update,
    // since TokenizeOperator needs to be involved.
    Collections.sort(indexes, (o1, o2) -> o1.getIndexType().ordinal() - o2.getIndexType().ordinal());
    // At this point, we have the data type info, and the indexes info as well
    int secondaryIndexTotalCnt = indexes.size() - 1;
    if (secondaryIndexTotalCnt > 0) {
        op0.getInputs().clear();
    } else {
        return false;
    }
    // Initialize inputs to the SINK operator Op0 (The SINK) is now without input
    // Prepare filtering field information (This is the filter created using the "filter with" key word in the
    // create dataset ddl)
    List<String> filteringFields = ((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterField();
    List<LogicalVariable> filteringVars;
    List<Mutable<ILogicalExpression>> filteringExpressions = null;
    if (filteringFields != null) {
        // The filter field var already exists. we can simply get it from the insert op
        filteringVars = new ArrayList<>();
        filteringExpressions = new ArrayList<>();
        for (Mutable<ILogicalExpression> filteringExpression : primaryIndexModificationOp.getAdditionalFilteringExpressions()) {
            filteringExpression.getValue().getUsedVariables(filteringVars);
            for (LogicalVariable var : filteringVars) {
                filteringExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(var)));
            }
        }
    }
    // Replicate Operator is applied only when doing the bulk-load.
    ReplicateOperator replicateOp = null;
    if (secondaryIndexTotalCnt > 1 && primaryIndexModificationOp.isBulkload()) {
        // Split the logical plan into "each secondary index update branch"
        // to replicate each <PK,OBJECT> pair.
        replicateOp = new ReplicateOperator(secondaryIndexTotalCnt);
        replicateOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
        replicateOp.setExecutionMode(ExecutionMode.PARTITIONED);
        context.computeAndSetTypeEnvironmentForOperator(replicateOp);
        currentTop = replicateOp;
    }
    /*
         * The two maps are used to store variables to which [casted] field access is assigned.
         * One for the beforeOp record and the other for the new record.
         * There are two uses for these maps:
         * 1. used for shared fields in indexes with overlapping keys.
         * 2. used for setting variables of secondary keys for each secondary index operator.
         */
    Map<IndexFieldId, LogicalVariable> fieldVarsForBeforeOperation = new HashMap<>();
    Map<IndexFieldId, LogicalVariable> fieldVarsForNewRecord = new HashMap<>();
    /*
         * if the index is enforcing field types (For open indexes), We add a cast
         * operator to ensure type safety
         */
    try {
        if (primaryIndexModificationOp.getOperation() == Kind.INSERT || primaryIndexModificationOp.getOperation() == Kind.UPSERT || /* Actually, delete should not be here but it is now until issue
                     * https://issues.apache.org/jira/browse/ASTERIXDB-1507
                     * is solved
                     */
        primaryIndexModificationOp.getOperation() == Kind.DELETE) {
            injectFieldAccessesForIndexes(context, dataset, indexes, fieldVarsForNewRecord, recType, metaType, newRecordVar, newMetaVar, primaryIndexModificationOp, false);
            if (replicateOp != null) {
                context.computeAndSetTypeEnvironmentForOperator(replicateOp);
            }
        }
        if (primaryIndexModificationOp.getOperation() == Kind.UPSERT) /* Actually, delete should be here but it is not until issue
             * https://issues.apache.org/jira/browse/ASTERIXDB-1507
             * is solved
             */
        {
            List<LogicalVariable> beforeOpMetaVars = primaryIndexModificationOp.getBeforeOpAdditionalNonFilteringVars();
            LogicalVariable beforeOpMetaVar = beforeOpMetaVars == null ? null : beforeOpMetaVars.get(0);
            currentTop = injectFieldAccessesForIndexes(context, dataset, indexes, fieldVarsForBeforeOperation, recType, metaType, primaryIndexModificationOp.getBeforeOpRecordVar(), beforeOpMetaVar, currentTop, true);
        }
    } catch (AsterixException e) {
        throw new AlgebricksException(e);
    }
    // At first, op1 is the index insert op insertOp
    for (Index index : indexes) {
        if (!index.isSecondaryIndex()) {
            continue;
        }
        hasSecondaryIndex = true;
        // Get the secondary fields names and types
        List<List<String>> secondaryKeyFields = index.getKeyFieldNames();
        List<LogicalVariable> secondaryKeyVars = new ArrayList<>();
        List<Mutable<ILogicalExpression>> secondaryExpressions = new ArrayList<>();
        List<Mutable<ILogicalExpression>> beforeOpSecondaryExpressions = new ArrayList<>();
        ILogicalOperator replicateOutput;
        for (int i = 0; i < secondaryKeyFields.size(); i++) {
            IndexFieldId indexFieldId = new IndexFieldId(index.getKeyFieldSourceIndicators().get(i), secondaryKeyFields.get(i));
            LogicalVariable skVar = fieldVarsForNewRecord.get(indexFieldId);
            secondaryKeyVars.add(skVar);
            secondaryExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(skVar)));
            if (primaryIndexModificationOp.getOperation() == Kind.UPSERT) {
                beforeOpSecondaryExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(fieldVarsForBeforeOperation.get(indexFieldId))));
            }
        }
        IndexInsertDeleteUpsertOperator indexUpdate;
        if (index.getIndexType() != IndexType.RTREE) {
            // Create an expression per key
            Mutable<ILogicalExpression> filterExpression = (primaryIndexModificationOp.getOperation() == Kind.UPSERT) ? null : createFilterExpression(secondaryKeyVars, context.getOutputTypeEnvironment(currentTop), index.isEnforcingKeyFileds());
            DataSourceIndex dataSourceIndex = new DataSourceIndex(index, dataverseName, datasetName, mp);
            // and index type is keyword or n-gram.
            if (index.getIndexType() != IndexType.BTREE && primaryIndexModificationOp.isBulkload()) {
                // Note: Bulk load case, we don't need to take care of it for upsert operation
                // Check whether the index is length-partitioned or not.
                // If partitioned, [input variables to TokenizeOperator,
                // token, number of token] pairs will be generated and
                // fed into the IndexInsertDeleteOperator.
                // If not, [input variables, token] pairs will be generated
                // and fed into the IndexInsertDeleteOperator.
                // Input variables are passed since TokenizeOperator is not an
                // filtering operator.
                boolean isPartitioned = index.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX || index.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX;
                // Create a new logical variable - token
                List<LogicalVariable> tokenizeKeyVars = new ArrayList<>();
                List<Mutable<ILogicalExpression>> tokenizeKeyExprs = new ArrayList<>();
                LogicalVariable tokenVar = context.newVar();
                tokenizeKeyVars.add(tokenVar);
                tokenizeKeyExprs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(tokenVar)));
                // Check the field type of the secondary key.
                IAType secondaryKeyType;
                Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0), secondaryKeyFields.get(0), recType);
                secondaryKeyType = keyPairType.first;
                List<Object> varTypes = new ArrayList<>();
                varTypes.add(NonTaggedFormatUtil.getTokenType(secondaryKeyType));
                // The type is short, and this does not contain type info.
                if (isPartitioned) {
                    LogicalVariable lengthVar = context.newVar();
                    tokenizeKeyVars.add(lengthVar);
                    tokenizeKeyExprs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(lengthVar)));
                    varTypes.add(BuiltinType.SHORTWITHOUTTYPEINFO);
                }
                // TokenizeOperator to tokenize [SK, PK] pairs
                TokenizeOperator tokenUpdate = new TokenizeOperator(dataSourceIndex, primaryIndexModificationOp.getPrimaryKeyExpressions(), secondaryExpressions, tokenizeKeyVars, filterExpression, primaryIndexModificationOp.getOperation(), primaryIndexModificationOp.isBulkload(), isPartitioned, varTypes);
                tokenUpdate.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
                context.computeAndSetTypeEnvironmentForOperator(tokenUpdate);
                replicateOutput = tokenUpdate;
                indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex, primaryIndexModificationOp.getPrimaryKeyExpressions(), tokenizeKeyExprs, filterExpression, primaryIndexModificationOp.getOperation(), primaryIndexModificationOp.isBulkload(), primaryIndexModificationOp.getAdditionalNonFilteringExpressions() == null ? 0 : primaryIndexModificationOp.getAdditionalNonFilteringExpressions().size());
                indexUpdate.setAdditionalFilteringExpressions(filteringExpressions);
                indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(tokenUpdate));
            } else {
                // When TokenizeOperator is not needed
                indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex, primaryIndexModificationOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression, primaryIndexModificationOp.getOperation(), primaryIndexModificationOp.isBulkload(), primaryIndexModificationOp.getAdditionalNonFilteringExpressions() == null ? 0 : primaryIndexModificationOp.getAdditionalNonFilteringExpressions().size());
                indexUpdate.setAdditionalFilteringExpressions(filteringExpressions);
                replicateOutput = indexUpdate;
                // We add the necessary expressions for upsert
                if (primaryIndexModificationOp.getOperation() == Kind.UPSERT) {
                    indexUpdate.setBeforeOpSecondaryKeyExprs(beforeOpSecondaryExpressions);
                    if (filteringFields != null) {
                        indexUpdate.setBeforeOpAdditionalFilteringExpression(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(primaryIndexModificationOp.getBeforeOpFilterVar())));
                    }
                }
                indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
            }
        } else {
            // Get type, dimensions and number of keys
            Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0), secondaryKeyFields.get(0), recType);
            IAType spatialType = keyPairType.first;
            boolean isPointMBR = spatialType.getTypeTag() == ATypeTag.POINT || spatialType.getTypeTag() == ATypeTag.POINT3D;
            int dimension = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
            int numKeys = (isPointMBR && isBulkload) ? dimension : dimension * 2;
            // Get variables and expressions
            List<LogicalVariable> keyVarList = new ArrayList<>();
            List<Mutable<ILogicalExpression>> keyExprList = new ArrayList<>();
            for (int i = 0; i < numKeys; i++) {
                LogicalVariable keyVar = context.newVar();
                keyVarList.add(keyVar);
                AbstractFunctionCallExpression createMBR = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.CREATE_MBR));
                createMBR.getArguments().add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVars.get(0))));
                createMBR.getArguments().add(new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AInt32(dimension)))));
                createMBR.getArguments().add(new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AInt32(i)))));
                keyExprList.add(new MutableObject<ILogicalExpression>(createMBR));
            }
            secondaryExpressions.clear();
            for (LogicalVariable secondaryKeyVar : keyVarList) {
                secondaryExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
            }
            if (isPointMBR && isBulkload) {
                //createFieldPermutationForBulkLoadOp(int) for more details.
                for (LogicalVariable secondaryKeyVar : keyVarList) {
                    secondaryExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
                }
            }
            AssignOperator assignCoordinates = new AssignOperator(keyVarList, keyExprList);
            assignCoordinates.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
            context.computeAndSetTypeEnvironmentForOperator(assignCoordinates);
            replicateOutput = assignCoordinates;
            Mutable<ILogicalExpression> filterExpression = null;
            AssignOperator originalAssignCoordinates = null;
            // We do something similar for beforeOp key if the operation is an upsert
            if (primaryIndexModificationOp.getOperation() == Kind.UPSERT) {
                List<LogicalVariable> originalKeyVarList = new ArrayList<>();
                List<Mutable<ILogicalExpression>> originalKeyExprList = new ArrayList<>();
                // we don't do any filtering since nulls are expected here and there
                for (int i = 0; i < numKeys; i++) {
                    LogicalVariable keyVar = context.newVar();
                    originalKeyVarList.add(keyVar);
                    AbstractFunctionCallExpression createMBR = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.CREATE_MBR));
                    createMBR.getArguments().add(beforeOpSecondaryExpressions.get(0));
                    createMBR.getArguments().add(new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AInt32(dimension)))));
                    createMBR.getArguments().add(new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AInt32(i)))));
                    originalKeyExprList.add(new MutableObject<ILogicalExpression>(createMBR));
                }
                beforeOpSecondaryExpressions.clear();
                for (LogicalVariable secondaryKeyVar : originalKeyVarList) {
                    beforeOpSecondaryExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
                }
                originalAssignCoordinates = new AssignOperator(originalKeyVarList, originalKeyExprList);
                originalAssignCoordinates.getInputs().add(new MutableObject<ILogicalOperator>(assignCoordinates));
                context.computeAndSetTypeEnvironmentForOperator(originalAssignCoordinates);
            } else {
                // We must enforce the filter if the originating spatial type is
                // nullable.
                boolean forceFilter = keyPairType.second;
                filterExpression = createFilterExpression(keyVarList, context.getOutputTypeEnvironment(assignCoordinates), forceFilter);
            }
            DataSourceIndex dataSourceIndex = new DataSourceIndex(index, dataverseName, datasetName, mp);
            indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex, primaryIndexModificationOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression, primaryIndexModificationOp.getOperation(), primaryIndexModificationOp.isBulkload(), primaryIndexModificationOp.getAdditionalNonFilteringExpressions() == null ? 0 : primaryIndexModificationOp.getAdditionalNonFilteringExpressions().size());
            indexUpdate.setAdditionalFilteringExpressions(filteringExpressions);
            if (primaryIndexModificationOp.getOperation() == Kind.UPSERT) {
                // set before op secondary key expressions
                if (filteringFields != null) {
                    indexUpdate.setBeforeOpAdditionalFilteringExpression(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(primaryIndexModificationOp.getBeforeOpFilterVar())));
                }
                // set filtering expressions
                indexUpdate.setBeforeOpSecondaryKeyExprs(beforeOpSecondaryExpressions);
                // assign --> assign beforeOp values --> secondary index upsert
                indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(originalAssignCoordinates));
            } else {
                indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(assignCoordinates));
            }
        }
        context.computeAndSetTypeEnvironmentForOperator(indexUpdate);
        if (!primaryIndexModificationOp.isBulkload() || secondaryIndexTotalCnt == 1) {
            currentTop = indexUpdate;
        } else {
            replicateOp.getOutputs().add(new MutableObject<>(replicateOutput));
        }
        if (primaryIndexModificationOp.isBulkload()) {
            // For bulk load, we connect all fanned out insert operator to a single SINK operator
            op0.getInputs().add(new MutableObject<ILogicalOperator>(indexUpdate));
        }
    }
    if (!hasSecondaryIndex) {
        return false;
    }
    if (!primaryIndexModificationOp.isBulkload()) {
        // If this is an upsert, we need to
        // Remove the current input to the SINK operator (It is actually already removed above)
        op0.getInputs().clear();
        // Connect the last index update to the SINK
        op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
    }
    return true;
}

Example 2 with CommitOperator

use of org.apache.asterix.algebra.operators.CommitOperator in project asterixdb by apache.

the class SetupCommitExtensionOpRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    if (op.getOperatorTag() != LogicalOperatorTag.DELEGATE_OPERATOR) {
        return false;
    }
    DelegateOperator eOp = (DelegateOperator) op;
    if (!(eOp.getDelegate() instanceof CommitOperator)) {
        return false;
    }
    boolean isSink = ((CommitOperator) eOp.getDelegate()).isSink();
    List<Mutable<ILogicalExpression>> primaryKeyExprs = null;
    Dataset dataset = null;
    AbstractLogicalOperator descendantOp = (AbstractLogicalOperator) eOp.getInputs().get(0).getValue();
    while (descendantOp != null) {
        if (descendantOp.getOperatorTag() == LogicalOperatorTag.INDEX_INSERT_DELETE_UPSERT) {
            IndexInsertDeleteUpsertOperator operator = (IndexInsertDeleteUpsertOperator) descendantOp;
            if (!operator.isBulkload() && operator.getPrevSecondaryKeyExprs() == null) {
                primaryKeyExprs = operator.getPrimaryKeyExpressions();
                dataset = ((DatasetDataSource) operator.getDataSourceIndex().getDataSource()).getDataset();
                break;
            }
        } else if (descendantOp.getOperatorTag() == LogicalOperatorTag.INSERT_DELETE_UPSERT) {
            InsertDeleteUpsertOperator insertDeleteUpsertOperator = (InsertDeleteUpsertOperator) descendantOp;
            if (!insertDeleteUpsertOperator.isBulkload()) {
                primaryKeyExprs = insertDeleteUpsertOperator.getPrimaryKeyExpressions();
                dataset = ((DatasetDataSource) insertDeleteUpsertOperator.getDataSource()).getDataset();
                break;
            }
        }
        if (descendantOp.getInputs().isEmpty()) {
            break;
        }
        descendantOp = (AbstractLogicalOperator) descendantOp.getInputs().get(0).getValue();
    }
    if (primaryKeyExprs == null) {
        return false;
    }
    //copy primaryKeyExprs
    List<LogicalVariable> primaryKeyLogicalVars = new ArrayList<>();
    for (Mutable<ILogicalExpression> expr : primaryKeyExprs) {
        VariableReferenceExpression varRefExpr = (VariableReferenceExpression) expr.getValue();
        primaryKeyLogicalVars.add(new LogicalVariable(varRefExpr.getVariableReference().getId()));
    }
    //get JobId(TransactorId)
    MetadataProvider mp = (MetadataProvider) context.getMetadataProvider();
    JobId jobId = mp.getJobId();
    //create the logical and physical operator
    CommitOperator commitOperator = new CommitOperator(primaryKeyLogicalVars, isSink);
    CommitPOperator commitPOperator = new CommitPOperator(jobId, dataset, primaryKeyLogicalVars, isSink);
    commitOperator.setPhysicalOperator(commitPOperator);
    //create ExtensionOperator and put the commitOperator in it.
    DelegateOperator extensionOperator = new DelegateOperator(commitOperator);
    extensionOperator.setPhysicalOperator(commitPOperator);
    //update plan link
    extensionOperator.getInputs().add(eOp.getInputs().get(0));
    context.computeAndSetTypeEnvironmentForOperator(extensionOperator);
    opRef.setValue(extensionOperator);
    return true;
}

Example 3 with CommitOperator

use of org.apache.asterix.algebra.operators.CommitOperator in project asterixdb by apache.

the class IntroduceDynamicTypeCastRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    // Depending on the operator type, we need to extract the following pieces of information.
    AbstractLogicalOperator op;
    ARecordType requiredRecordType;
    LogicalVariable recordVar;
    // We identify INSERT and DISTRIBUTE_RESULT operators.
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
    switch(op1.getOperatorTag()) {
        case SINK:
        case DELEGATE_OPERATOR:
            {
                /**
                 * pattern match: commit insert assign
                 * resulting plan: commit-insert-project-assign
                 */
                if (op1.getOperatorTag() == LogicalOperatorTag.DELEGATE_OPERATOR) {
                    DelegateOperator eOp = (DelegateOperator) op1;
                    if (!(eOp.getDelegate() instanceof CommitOperator)) {
                        return false;
                    }
                }
                AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
                if (op2.getOperatorTag() == LogicalOperatorTag.INSERT_DELETE_UPSERT) {
                    InsertDeleteUpsertOperator insertDeleteOp = (InsertDeleteUpsertOperator) op2;
                    if (insertDeleteOp.getOperation() == InsertDeleteUpsertOperator.Kind.DELETE) {
                        return false;
                    }
                    // Remember this is the operator we need to modify
                    op = insertDeleteOp;
                    // Derive the required ARecordType based on the schema of the DataSource
                    InsertDeleteUpsertOperator insertDeleteOperator = (InsertDeleteUpsertOperator) op2;
                    DataSource dataSource = (DataSource) insertDeleteOperator.getDataSource();
                    requiredRecordType = (ARecordType) dataSource.getItemType();
                    // Derive the Variable which we will potentially wrap with cast/null functions
                    ILogicalExpression expr = insertDeleteOperator.getPayloadExpression().getValue();
                    List<LogicalVariable> payloadVars = new ArrayList<>();
                    expr.getUsedVariables(payloadVars);
                    recordVar = payloadVars.get(0);
                } else {
                    return false;
                }
                break;
            }
        case DISTRIBUTE_RESULT:
            {
                // First, see if there was an output-record-type specified
                requiredRecordType = (ARecordType) op1.getAnnotations().get("output-record-type");
                if (requiredRecordType == null) {
                    return false;
                }
                // Remember this is the operator we need to modify
                op = op1;
                recordVar = ((VariableReferenceExpression) ((DistributeResultOperator) op).getExpressions().get(0).getValue()).getVariableReference();
                break;
            }
        default:
            {
                return false;
            }
    }
    // Derive the statically-computed type of the record
    IVariableTypeEnvironment env = op.computeOutputTypeEnvironment(context);
    IAType inputRecordType = (IAType) env.getVarType(recordVar);
    /** the input record type can be an union type -- for the case when it comes from a subplan or left-outer join */
    boolean checkUnknown = false;
    while (NonTaggedFormatUtil.isOptional(inputRecordType)) {
        /** while-loop for the case there is a nested multi-level union */
        inputRecordType = ((AUnionType) inputRecordType).getActualType();
        checkUnknown = true;
    }
    /** see whether the input record type needs to be casted */
    boolean cast = !compatible(requiredRecordType, inputRecordType);
    if (checkUnknown) {
        recordVar = addWrapperFunction(requiredRecordType, recordVar, op, context, BuiltinFunctions.CHECK_UNKNOWN);
    }
    if (cast) {
        addWrapperFunction(requiredRecordType, recordVar, op, context, BuiltinFunctions.CAST_TYPE);
    }
    return cast || checkUnknown;
}

Example 4 with CommitOperator

use of org.apache.asterix.algebra.operators.CommitOperator in project asterixdb by apache.

the class LangExpressionToPlanTranslator method translateSubscribeFeed.

private ILogicalOperator translateSubscribeFeed(CompiledSubscribeFeedStatement sfs, DatasetDataSource targetDatasource, LogicalVariable unnestVar, ILogicalOperator topOp, ArrayList<Mutable<ILogicalExpression>> exprs, LogicalVariable resVar, List<Mutable<ILogicalExpression>> varRefsForLoading, Mutable<ILogicalExpression> varRef, ILogicalOperator assign, List<String> additionalFilteringField, AssignOperator additionalFilteringAssign, List<Mutable<ILogicalExpression>> additionalFilteringExpressions) throws AlgebricksException {
    // if the feed is a change feed (i.e, performs different operations), we need to project op variable
    InsertDeleteUpsertOperator feedModificationOp;
    AssignOperator metaAndKeysAssign;
    List<LogicalVariable> metaAndKeysVars = null;
    List<Mutable<ILogicalExpression>> metaAndKeysExprs = null;
    List<Mutable<ILogicalExpression>> metaExpSingletonList = null;
    Feed feed = metadataProvider.findFeed(sfs.getDataverseName(), sfs.getFeedName());
    boolean isChangeFeed = ExternalDataUtils.isChangeFeed(feed.getAdapterConfiguration());
    boolean isUpsertFeed = ExternalDataUtils.isUpsertFeed(feed.getAdapterConfiguration());
    ProjectOperator project = (ProjectOperator) topOp;
    if (targetDatasource.getDataset().hasMetaPart() || isChangeFeed) {
        metaAndKeysVars = new ArrayList<>();
        metaAndKeysExprs = new ArrayList<>();
        if (targetDatasource.getDataset().hasMetaPart()) {
            // add the meta function
            IFunctionInfo finfoMeta = FunctionUtil.getFunctionInfo(BuiltinFunctions.META);
            ScalarFunctionCallExpression metaFunction = new ScalarFunctionCallExpression(finfoMeta, new MutableObject<>(new VariableReferenceExpression(unnestVar)));
            // create assign for the meta part
            LogicalVariable metaVar = context.newVar();
            metaExpSingletonList = new ArrayList<>(1);
            metaExpSingletonList.add(new MutableObject<>(new VariableReferenceExpression(metaVar)));
            metaAndKeysVars.add(metaVar);
            metaAndKeysExprs.add(new MutableObject<>(metaFunction));
            project.getVariables().add(metaVar);
        }
    }
    if (isChangeFeed) {
        varRefsForLoading.clear();
        for (Mutable<ILogicalExpression> assignExpr : exprs) {
            if (assignExpr.getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
                AbstractFunctionCallExpression funcCall = (AbstractFunctionCallExpression) assignExpr.getValue();
                funcCall.substituteVar(resVar, unnestVar);
                LogicalVariable pkVar = context.newVar();
                metaAndKeysVars.add(pkVar);
                metaAndKeysExprs.add(new MutableObject<>(assignExpr.getValue()));
                project.getVariables().add(pkVar);
                varRefsForLoading.add(new MutableObject<>(new VariableReferenceExpression(pkVar)));
            }
        }
        // A change feed, we don't need the assign to access PKs
        feedModificationOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading, metaExpSingletonList, InsertDeleteUpsertOperator.Kind.UPSERT, false);
        // Create and add a new variable used for representing the original record
        feedModificationOp.setPrevRecordVar(context.newVar());
        feedModificationOp.setPrevRecordType(targetDatasource.getItemType());
        if (targetDatasource.getDataset().hasMetaPart()) {
            List<LogicalVariable> metaVars = new ArrayList<>();
            metaVars.add(context.newVar());
            feedModificationOp.setPrevAdditionalNonFilteringVars(metaVars);
            List<Object> metaTypes = new ArrayList<>();
            metaTypes.add(targetDatasource.getMetaItemType());
            feedModificationOp.setPrevAdditionalNonFilteringTypes(metaTypes);
        }
        if (additionalFilteringField != null) {
            feedModificationOp.setPrevFilterVar(context.newVar());
            feedModificationOp.setPrevFilterType(((ARecordType) targetDatasource.getItemType()).getFieldType(additionalFilteringField.get(0)));
            additionalFilteringAssign.getInputs().clear();
            additionalFilteringAssign.getInputs().add(assign.getInputs().get(0));
            feedModificationOp.getInputs().add(new MutableObject<>(additionalFilteringAssign));
        } else {
            feedModificationOp.getInputs().add(assign.getInputs().get(0));
        }
    } else {
        final InsertDeleteUpsertOperator.Kind opKind = isUpsertFeed ? InsertDeleteUpsertOperator.Kind.UPSERT : InsertDeleteUpsertOperator.Kind.INSERT;
        feedModificationOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading, metaExpSingletonList, opKind, false);
        if (isUpsertFeed) {
            feedModificationOp.setPrevRecordVar(context.newVar());
            feedModificationOp.setPrevRecordType(targetDatasource.getItemType());
        }
        feedModificationOp.getInputs().add(new MutableObject<>(assign));
    }
    if (targetDatasource.getDataset().hasMetaPart() || isChangeFeed) {
        metaAndKeysAssign = new AssignOperator(metaAndKeysVars, metaAndKeysExprs);
        metaAndKeysAssign.getInputs().add(topOp.getInputs().get(0));
        topOp.getInputs().set(0, new MutableObject<>(metaAndKeysAssign));
    }
    feedModificationOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
    ILogicalOperator leafOperator = new DelegateOperator(new CommitOperator(true));
    leafOperator.getInputs().add(new MutableObject<>(feedModificationOp));
    return leafOperator;
}

Example 5 with CommitOperator

use of org.apache.asterix.algebra.operators.CommitOperator in project asterixdb by apache.

the class IntroduceRapidFrameFlushProjectAssignRule method checkIfRuleIsApplicable.

private boolean checkIfRuleIsApplicable(AbstractLogicalOperator op) {
    if (op.getOperatorTag() != LogicalOperatorTag.DELEGATE_OPERATOR) {
        return false;
    }
    DelegateOperator extensionOp = (DelegateOperator) op;
    if (!(extensionOp.getDelegate() instanceof CommitOperator)) {
        return false;
    }
    for (int i = 0; i < op.getInputs().size(); ++i) {
        AbstractLogicalOperator descendantOp = (AbstractLogicalOperator) op.getInputs().get(i).getValue();
        if (descendantOp.getOperatorTag() == LogicalOperatorTag.PROJECT || descendantOp.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
            if (descendantOp.getPhysicalOperator() == null) {
                return false;
            }
        }
        checkIfRuleIsApplicable(descendantOp);
    }
    return true;
}