Examples with DataSourceScanOperator org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator used on opensource projects

Example 16 with DataSourceScanOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator in project asterixdb by apache.

the class OptimizableOperatorSubTree method setDatasetAndTypeMetadata.

/**
     * Find the dataset corresponding to the datasource scan in the metadata.
     * Also sets recordType to be the type of that dataset.
     */
public boolean setDatasetAndTypeMetadata(MetadataProvider metadataProvider) throws AlgebricksException {
    String dataverseName = null;
    String datasetName = null;
    Dataset ds = null;
    ARecordType rType = null;
    List<Mutable<ILogicalOperator>> sourceOpRefs = new ArrayList<>();
    List<DataSourceType> dsTypes = new ArrayList<>();
    sourceOpRefs.add(getDataSourceRef());
    dsTypes.add(getDataSourceType());
    // If there are multiple datasources in the subtree, we need to find the dataset for these.
    if (getIxJoinOuterAdditionalDataSourceRefs() != null) {
        for (int i = 0; i < getIxJoinOuterAdditionalDataSourceRefs().size(); i++) {
            sourceOpRefs.add(getIxJoinOuterAdditionalDataSourceRefs().get(i));
            dsTypes.add(getIxJoinOuterAdditionalDataSourceTypes().get(i));
        }
    }
    for (int i = 0; i < sourceOpRefs.size(); i++) {
        switch(dsTypes.get(i)) {
            case DATASOURCE_SCAN:
                DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) sourceOpRefs.get(i).getValue();
                IDataSource<?> datasource = dataSourceScan.getDataSource();
                if (datasource instanceof DataSource) {
                    byte dsType = ((DataSource) datasource).getDatasourceType();
                    if (dsType != DataSource.Type.INTERNAL_DATASET && dsType != DataSource.Type.EXTERNAL_DATASET) {
                        return false;
                    }
                }
                Pair<String, String> datasetInfo = AnalysisUtil.getDatasetInfo(dataSourceScan);
                dataverseName = datasetInfo.first;
                datasetName = datasetInfo.second;
                break;
            case PRIMARY_INDEX_LOOKUP:
                AbstractUnnestOperator unnestMapOp = (AbstractUnnestOperator) sourceOpRefs.get(i).getValue();
                ILogicalExpression unnestExpr = unnestMapOp.getExpressionRef().getValue();
                AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
                AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
                jobGenParams.readFromFuncArgs(f.getArguments());
                datasetName = jobGenParams.getDatasetName();
                dataverseName = jobGenParams.getDataverseName();
                break;
            case EXTERNAL_SCAN:
                UnnestMapOperator externalScan = (UnnestMapOperator) sourceOpRefs.get(i).getValue();
                datasetInfo = AnalysisUtil.getExternalDatasetInfo(externalScan);
                dataverseName = datasetInfo.first;
                datasetName = datasetInfo.second;
                break;
            case COLLECTION_SCAN:
                if (i != 0) {
                    getIxJoinOuterAdditionalDatasets().add(null);
                    getIxJoinOuterAdditionalRecordTypes().add(null);
                }
                continue;
            case NO_DATASOURCE:
            default:
                return false;
        }
        if (dataverseName == null || datasetName == null) {
            return false;
        }
        // Find the dataset corresponding to the datasource in the metadata.
        ds = metadataProvider.findDataset(dataverseName, datasetName);
        if (ds == null) {
            throw CompilationException.create(ErrorCode.NO_METADATA_FOR_DATASET, datasetName);
        }
        // Get the record type for that dataset.
        IAType itemType = metadataProvider.findType(ds.getItemTypeDataverseName(), ds.getItemTypeName());
        if (itemType.getTypeTag() != ATypeTag.OBJECT) {
            if (i == 0) {
                return false;
            } else {
                getIxJoinOuterAdditionalDatasets().add(null);
                getIxJoinOuterAdditionalRecordTypes().add(null);
            }
        }
        rType = (ARecordType) itemType;
        // Get the meta record type for that dataset.
        IAType metaItemType = metadataProvider.findType(ds.getMetaItemTypeDataverseName(), ds.getMetaItemTypeName());
        // First index is always the primary datasource in this subtree.
        if (i == 0) {
            setDataset(ds);
            setRecordType(rType);
            setMetaRecordType((ARecordType) metaItemType);
        } else {
            getIxJoinOuterAdditionalDatasets().add(ds);
            getIxJoinOuterAdditionalRecordTypes().add(rType);
        }
        dataverseName = null;
        datasetName = null;
        ds = null;
        rType = null;
    }
    return true;
}

Example 17 with DataSourceScanOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator in project asterixdb by apache.

the class InvertedIndexAccessMethod method createSecondaryToPrimaryPlan.

@Override
public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef, OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context) throws AlgebricksException {
    IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
    Dataset dataset = indexSubTree.getDataset();
    ARecordType recordType = indexSubTree.getRecordType();
    ARecordType metaRecordType = indexSubTree.getMetaRecordType();
    // we made sure indexSubTree has datasource scan
    DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) indexSubTree.getDataSourceRef().getValue();
    InvertedIndexJobGenParams jobGenParams = new InvertedIndexJobGenParams(chosenIndex.getIndexName(), chosenIndex.getIndexType(), dataset.getDataverseName(), dataset.getDatasetName(), retainInput, requiresBroadcast);
    // Add function-specific args such as search modifier, and possibly a similarity threshold.
    addFunctionSpecificArgs(optFuncExpr, jobGenParams);
    // Add the type of search key from the optFuncExpr.
    addSearchKeyType(optFuncExpr, indexSubTree, context, jobGenParams);
    // Operator that feeds the secondary-index search.
    AbstractLogicalOperator inputOp = null;
    // Here we generate vars and funcs for assigning the secondary-index keys to be fed into the secondary-index search.
    // List of variables for the assign.
    ArrayList<LogicalVariable> keyVarList = new ArrayList<LogicalVariable>();
    // probeSubTree is null if we are dealing with a selection query, and non-null for join queries.
    if (probeSubTree == null) {
        // List of expressions for the assign.
        ArrayList<Mutable<ILogicalExpression>> keyExprList = new ArrayList<Mutable<ILogicalExpression>>();
        // Add key vars and exprs to argument list.
        addKeyVarsAndExprs(optFuncExpr, keyVarList, keyExprList, context);
        // Assign operator that sets the secondary-index search-key fields.
        inputOp = new AssignOperator(keyVarList, keyExprList);
        // Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input).
        inputOp.getInputs().add(new MutableObject<>(OperatorManipulationUtil.deepCopy(dataSourceScan.getInputs().get(0).getValue())));
        inputOp.setExecutionMode(dataSourceScan.getExecutionMode());
    } else {
        // We are optimizing a join. Add the input variable to the secondaryIndexFuncArgs.
        LogicalVariable inputSearchVariable = getInputSearchVar(optFuncExpr, indexSubTree);
        keyVarList.add(inputSearchVariable);
        inputOp = (AbstractLogicalOperator) probeSubTree.getRoot();
    }
    jobGenParams.setKeyVarList(keyVarList);
    ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType, metaRecordType, chosenIndex, inputOp, jobGenParams, context, true, retainInput, retainNull);
    // Generate the rest of the upstream plan which feeds the search results into the primary index.
    AbstractUnnestMapOperator primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceScan, dataset, recordType, metaRecordType, secondaryIndexUnnestOp, context, true, retainInput, retainNull, false);
    return primaryIndexUnnestOp;
}

Example 18 with DataSourceScanOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator in project asterixdb by apache.

the class IntroduceLSMComponentFilterRule method changePlan.

private void changePlan(List<IOptimizableFuncExpr> optFuncExprs, AbstractLogicalOperator op, Dataset dataset, IOptimizationContext context) throws AlgebricksException {
    Queue<Mutable<ILogicalOperator>> queue = new LinkedList<>(op.getInputs());
    while (!queue.isEmpty()) {
        AbstractLogicalOperator descendantOp = (AbstractLogicalOperator) queue.poll().getValue();
        if (descendantOp == null) {
            continue;
        }
        if (descendantOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
            DataSourceScanOperator dataSourceScanOp = (DataSourceScanOperator) descendantOp;
            DataSource ds = (DataSource) dataSourceScanOp.getDataSource();
            if (dataset.getDatasetName().compareTo(((DatasetDataSource) ds).getDataset().getDatasetName()) == 0) {
                List<LogicalVariable> minFilterVars = new ArrayList<>();
                List<LogicalVariable> maxFilterVars = new ArrayList<>();
                AssignOperator assignOp = createAssignOperator(optFuncExprs, minFilterVars, maxFilterVars, context);
                dataSourceScanOp.setMinFilterVars(minFilterVars);
                dataSourceScanOp.setMaxFilterVars(maxFilterVars);
                List<Mutable<ILogicalExpression>> additionalFilteringExpressions = new ArrayList<>();
                for (LogicalVariable var : assignOp.getVariables()) {
                    additionalFilteringExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(var)));
                }
                dataSourceScanOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
                assignOp.getInputs().add(new MutableObject<>(dataSourceScanOp.getInputs().get(0).getValue()));
                dataSourceScanOp.getInputs().get(0).setValue(assignOp);
            }
        } else if (descendantOp.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) {
            UnnestMapOperator unnestMapOp = (UnnestMapOperator) descendantOp;
            ILogicalExpression unnestExpr = unnestMapOp.getExpressionRef().getValue();
            if (unnestExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
                AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
                FunctionIdentifier fid = f.getFunctionIdentifier();
                if (!fid.equals(BuiltinFunctions.INDEX_SEARCH)) {
                    throw new IllegalStateException();
                }
                AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
                jobGenParams.readFromFuncArgs(f.getArguments());
                if (dataset.getDatasetName().compareTo(jobGenParams.datasetName) == 0) {
                    List<LogicalVariable> minFilterVars = new ArrayList<>();
                    List<LogicalVariable> maxFilterVars = new ArrayList<>();
                    AssignOperator assignOp = createAssignOperator(optFuncExprs, minFilterVars, maxFilterVars, context);
                    unnestMapOp.setMinFilterVars(minFilterVars);
                    unnestMapOp.setMaxFilterVars(maxFilterVars);
                    List<Mutable<ILogicalExpression>> additionalFilteringExpressions = new ArrayList<>();
                    for (LogicalVariable var : assignOp.getVariables()) {
                        additionalFilteringExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(var)));
                    }
                    unnestMapOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
                    assignOp.getInputs().add(new MutableObject<>(unnestMapOp.getInputs().get(0).getValue()));
                    unnestMapOp.getInputs().get(0).setValue(assignOp);
                }
            }
        }
        queue.addAll(descendantOp.getInputs());
    }
}

Example 19 with DataSourceScanOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator in project asterixdb by apache.

the class IsomorphismOperatorVisitor method visitDataScanOperator.

@Override
public Boolean visitDataScanOperator(DataSourceScanOperator op, ILogicalOperator arg) throws AlgebricksException {
    AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
    if (aop.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN) {
        return Boolean.FALSE;
    }
    DataSourceScanOperator argScan = (DataSourceScanOperator) arg;
    if (!argScan.getDataSource().toString().equals(op.getDataSource().toString())) {
        return Boolean.FALSE;
    }
    DataSourceScanOperator scanOpArg = (DataSourceScanOperator) copyAndSubstituteVar(op, arg);
    boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), scanOpArg.getVariables()) && op.getDataSource().toString().equals(scanOpArg.getDataSource().toString());
    return isomorphic;
}

Example 20 with DataSourceScanOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator in project asterixdb by apache.

the class DataSourceScanPOperator method computeDeliveredProperties.

@Override
public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
    // partitioning properties
    DataSourceScanOperator dssOp = (DataSourceScanOperator) op;
    IDataSourcePropertiesProvider dspp = dataSource.getPropertiesProvider();
    deliveredProperties = dspp.computePropertiesVector(dssOp.getVariables());
}