Examples with IDataSource org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource used on opensource projects

Example 1 with IDataSource

use of org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource in project asterixdb by apache.

the class PushFieldAccessRule method propagateFieldAccessRec.

@SuppressWarnings("unchecked")
private boolean propagateFieldAccessRec(Mutable<ILogicalOperator> opRef, IOptimizationContext context, String finalAnnot) throws AlgebricksException {
    AssignOperator access = (AssignOperator) opRef.getValue();
    Mutable<ILogicalOperator> opRef2 = access.getInputs().get(0);
    AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
    // rewritten into index search.
    if (op2.getOperatorTag() == LogicalOperatorTag.PROJECT || context.checkAndAddToAlreadyCompared(access, op2) && !(op2.getOperatorTag() == LogicalOperatorTag.SELECT && isAccessToIndexedField(access, context))) {
        return false;
    }
    Object annotation = op2.getAnnotations().get(IS_MOVABLE);
    if (annotation != null && !((Boolean) annotation)) {
        return false;
    }
    if (tryingToPushThroughSelectionWithSameDataSource(access, op2)) {
        return false;
    }
    if (testAndModifyRedundantOp(access, op2)) {
        propagateFieldAccessRec(opRef2, context, finalAnnot);
        return true;
    }
    List<LogicalVariable> usedInAccess = new LinkedList<>();
    VariableUtilities.getUsedVariables(access, usedInAccess);
    List<LogicalVariable> produced2 = new LinkedList<>();
    if (op2.getOperatorTag() == LogicalOperatorTag.GROUP) {
        VariableUtilities.getLiveVariables(op2, produced2);
    } else {
        VariableUtilities.getProducedVariables(op2, produced2);
    }
    boolean pushItDown = false;
    List<LogicalVariable> inter = new ArrayList<>(usedInAccess);
    if (inter.isEmpty()) {
        // ground value
        return false;
    }
    inter.retainAll(produced2);
    if (inter.isEmpty()) {
        pushItDown = true;
    } else if (op2.getOperatorTag() == LogicalOperatorTag.GROUP) {
        GroupByOperator g = (GroupByOperator) op2;
        List<Pair<LogicalVariable, LogicalVariable>> varMappings = new ArrayList<>();
        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : g.getDecorList()) {
            ILogicalExpression e = p.second.getValue();
            if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                LogicalVariable decorVar = GroupByOperator.getDecorVariable(p);
                if (inter.contains(decorVar)) {
                    inter.remove(decorVar);
                    LogicalVariable v1 = ((VariableReferenceExpression) e).getVariableReference();
                    varMappings.add(new Pair<>(decorVar, v1));
                }
            }
        }
        if (inter.isEmpty()) {
            boolean changed = false;
            for (Pair<LogicalVariable, LogicalVariable> m : varMappings) {
                LogicalVariable v2 = context.newVar();
                LogicalVariable oldVar = access.getVariables().get(0);
                g.getDecorList().add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(oldVar, new MutableObject<ILogicalExpression>(new VariableReferenceExpression(v2))));
                changed = true;
                access.getVariables().set(0, v2);
                VariableUtilities.substituteVariables(access, m.first, m.second, context);
            }
            if (changed) {
                context.computeAndSetTypeEnvironmentForOperator(g);
            }
            usedInAccess.clear();
            VariableUtilities.getUsedVariables(access, usedInAccess);
            pushItDown = true;
        }
    }
    if (pushItDown) {
        if (op2.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
            Mutable<ILogicalOperator> childOfSubplan = ((NestedTupleSourceOperator) op2).getDataSourceReference().getValue().getInputs().get(0);
            pushAccessDown(opRef, op2, childOfSubplan, context, finalAnnot);
            return true;
        }
        if (op2.getInputs().size() == 1 && !op2.hasNestedPlans()) {
            pushAccessDown(opRef, op2, op2.getInputs().get(0), context, finalAnnot);
            return true;
        } else {
            for (Mutable<ILogicalOperator> inp : op2.getInputs()) {
                HashSet<LogicalVariable> v2 = new HashSet<>();
                VariableUtilities.getLiveVariables(inp.getValue(), v2);
                if (v2.containsAll(usedInAccess)) {
                    pushAccessDown(opRef, op2, inp, context, finalAnnot);
                    return true;
                }
            }
        }
        if (op2.hasNestedPlans()) {
            AbstractOperatorWithNestedPlans nestedOp = (AbstractOperatorWithNestedPlans) op2;
            for (ILogicalPlan plan : nestedOp.getNestedPlans()) {
                for (Mutable<ILogicalOperator> root : plan.getRoots()) {
                    HashSet<LogicalVariable> v2 = new HashSet<>();
                    VariableUtilities.getLiveVariables(root.getValue(), v2);
                    if (v2.containsAll(usedInAccess)) {
                        pushAccessDown(opRef, op2, root, context, finalAnnot);
                        return true;
                    }
                }
            }
        }
        throw new AlgebricksException("Field access " + access.getExpressions().get(0).getValue() + " does not correspond to any input of operator " + op2);
    } else {
        // fields. If yes, we can equate the two variables.
        if (op2.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
            DataSourceScanOperator scan = (DataSourceScanOperator) op2;
            int n = scan.getVariables().size();
            LogicalVariable scanRecordVar = scan.getVariables().get(n - 1);
            AbstractFunctionCallExpression accessFun = (AbstractFunctionCallExpression) access.getExpressions().get(0).getValue();
            ILogicalExpression e0 = accessFun.getArguments().get(0).getValue();
            LogicalExpressionTag tag = e0.getExpressionTag();
            if (tag == LogicalExpressionTag.VARIABLE) {
                VariableReferenceExpression varRef = (VariableReferenceExpression) e0;
                if (varRef.getVariableReference() == scanRecordVar) {
                    ILogicalExpression e1 = accessFun.getArguments().get(1).getValue();
                    if (e1.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
                        IDataSource<DataSourceId> dataSource = (IDataSource<DataSourceId>) scan.getDataSource();
                        byte dsType = ((DataSource) dataSource).getDatasourceType();
                        if (dsType == DataSource.Type.FEED || dsType == DataSource.Type.LOADABLE) {
                            return false;
                        }
                        DataSourceId asid = dataSource.getId();
                        MetadataProvider mp = (MetadataProvider) context.getMetadataProvider();
                        Dataset dataset = mp.findDataset(asid.getDataverseName(), asid.getDatasourceName());
                        if (dataset == null) {
                            throw new AlgebricksException("Dataset " + asid.getDatasourceName() + " not found.");
                        }
                        if (dataset.getDatasetType() != DatasetType.INTERNAL) {
                            setAsFinal(access, context, finalAnnot);
                            return false;
                        }
                        ConstantExpression ce = (ConstantExpression) e1;
                        IAObject obj = ((AsterixConstantValue) ce.getValue()).getObject();
                        String fldName;
                        if (obj.getType().getTypeTag() == ATypeTag.STRING) {
                            fldName = ((AString) obj).getStringValue();
                        } else {
                            int pos = ((AInt32) obj).getIntegerValue();
                            String tName = dataset.getItemTypeName();
                            IAType t = mp.findType(dataset.getItemTypeDataverseName(), tName);
                            if (t.getTypeTag() != ATypeTag.OBJECT) {
                                return false;
                            }
                            ARecordType rt = (ARecordType) t;
                            if (pos >= rt.getFieldNames().length) {
                                setAsFinal(access, context, finalAnnot);
                                return false;
                            }
                            fldName = rt.getFieldNames()[pos];
                        }
                        int p = DatasetUtil.getPositionOfPartitioningKeyField(dataset, fldName);
                        if (p < 0) {
                            // not one of the partitioning fields
                            setAsFinal(access, context, finalAnnot);
                            return false;
                        }
                        LogicalVariable keyVar = scan.getVariables().get(p);
                        access.getExpressions().get(0).setValue(new VariableReferenceExpression(keyVar));
                        return true;
                    }
                }
            }
        }
        setAsFinal(access, context, finalAnnot);
        return false;
    }
}

Example 2 with IDataSource

use of org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource in project asterixdb by apache.

the class PushFieldAccessRule method isAccessToIndexedField.

@SuppressWarnings("unchecked")
private boolean isAccessToIndexedField(AssignOperator assign, IOptimizationContext context) throws AlgebricksException {
    AbstractFunctionCallExpression accessFun = (AbstractFunctionCallExpression) assign.getExpressions().get(0).getValue();
    ILogicalExpression e0 = accessFun.getArguments().get(0).getValue();
    if (e0.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
        return false;
    }
    LogicalVariable var = ((VariableReferenceExpression) e0).getVariableReference();
    if (context.findPrimaryKey(var) == null) {
        // not referring to a dataset record
        return false;
    }
    AbstractLogicalOperator op = assign;
    while (op.getInputs().size() == 1 && op.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN) {
        op = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
    }
    if (op.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN) {
        return false;
    }
    DataSourceScanOperator scan = (DataSourceScanOperator) op;
    LogicalVariable recVar = scan.getVariables().get(scan.getVariables().size() - 1);
    if (recVar != var) {
        return false;
    }
    MetadataProvider mp = (MetadataProvider) context.getMetadataProvider();
    DataSourceId asid = ((IDataSource<DataSourceId>) scan.getDataSource()).getId();
    Dataset dataset = mp.findDataset(asid.getDataverseName(), asid.getDatasourceName());
    if (dataset == null) {
        throw new AlgebricksException("Dataset " + asid.getDatasourceName() + " not found.");
    }
    if (dataset.getDatasetType() != DatasetType.INTERNAL) {
        return false;
    }
    final Integer pos = ConstantExpressionUtil.getIntConstant(accessFun.getArguments().get(1).getValue());
    if (pos != null) {
        String tName = dataset.getItemTypeName();
        IAType t = mp.findType(dataset.getItemTypeDataverseName(), tName);
        if (t.getTypeTag() != ATypeTag.OBJECT) {
            return false;
        }
        ARecordType rt = (ARecordType) t;
        if (pos >= rt.getFieldNames().length) {
            return false;
        }
    }
    List<Index> datasetIndexes = mp.getDatasetIndexes(dataset.getDataverseName(), dataset.getDatasetName());
    boolean hasSecondaryIndex = false;
    for (Index index : datasetIndexes) {
        if (index.isSecondaryIndex()) {
            hasSecondaryIndex = true;
            break;
        }
    }
    return hasSecondaryIndex;
}

Example 3 with IDataSource

use of org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource in project asterixdb by apache.

the class OptimizableOperatorSubTree method setDatasetAndTypeMetadata.

/**
     * Find the dataset corresponding to the datasource scan in the metadata.
     * Also sets recordType to be the type of that dataset.
     */
public boolean setDatasetAndTypeMetadata(MetadataProvider metadataProvider) throws AlgebricksException {
    String dataverseName = null;
    String datasetName = null;
    Dataset ds = null;
    ARecordType rType = null;
    List<Mutable<ILogicalOperator>> sourceOpRefs = new ArrayList<>();
    List<DataSourceType> dsTypes = new ArrayList<>();
    sourceOpRefs.add(getDataSourceRef());
    dsTypes.add(getDataSourceType());
    // If there are multiple datasources in the subtree, we need to find the dataset for these.
    if (getIxJoinOuterAdditionalDataSourceRefs() != null) {
        for (int i = 0; i < getIxJoinOuterAdditionalDataSourceRefs().size(); i++) {
            sourceOpRefs.add(getIxJoinOuterAdditionalDataSourceRefs().get(i));
            dsTypes.add(getIxJoinOuterAdditionalDataSourceTypes().get(i));
        }
    }
    for (int i = 0; i < sourceOpRefs.size(); i++) {
        switch(dsTypes.get(i)) {
            case DATASOURCE_SCAN:
                DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) sourceOpRefs.get(i).getValue();
                IDataSource<?> datasource = dataSourceScan.getDataSource();
                if (datasource instanceof DataSource) {
                    byte dsType = ((DataSource) datasource).getDatasourceType();
                    if (dsType != DataSource.Type.INTERNAL_DATASET && dsType != DataSource.Type.EXTERNAL_DATASET) {
                        return false;
                    }
                }
                Pair<String, String> datasetInfo = AnalysisUtil.getDatasetInfo(dataSourceScan);
                dataverseName = datasetInfo.first;
                datasetName = datasetInfo.second;
                break;
            case PRIMARY_INDEX_LOOKUP:
                AbstractUnnestOperator unnestMapOp = (AbstractUnnestOperator) sourceOpRefs.get(i).getValue();
                ILogicalExpression unnestExpr = unnestMapOp.getExpressionRef().getValue();
                AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
                AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
                jobGenParams.readFromFuncArgs(f.getArguments());
                datasetName = jobGenParams.getDatasetName();
                dataverseName = jobGenParams.getDataverseName();
                break;
            case EXTERNAL_SCAN:
                UnnestMapOperator externalScan = (UnnestMapOperator) sourceOpRefs.get(i).getValue();
                datasetInfo = AnalysisUtil.getExternalDatasetInfo(externalScan);
                dataverseName = datasetInfo.first;
                datasetName = datasetInfo.second;
                break;
            case COLLECTION_SCAN:
                if (i != 0) {
                    getIxJoinOuterAdditionalDatasets().add(null);
                    getIxJoinOuterAdditionalRecordTypes().add(null);
                }
                continue;
            case NO_DATASOURCE:
            default:
                return false;
        }
        if (dataverseName == null || datasetName == null) {
            return false;
        }
        // Find the dataset corresponding to the datasource in the metadata.
        ds = metadataProvider.findDataset(dataverseName, datasetName);
        if (ds == null) {
            throw CompilationException.create(ErrorCode.NO_METADATA_FOR_DATASET, datasetName);
        }
        // Get the record type for that dataset.
        IAType itemType = metadataProvider.findType(ds.getItemTypeDataverseName(), ds.getItemTypeName());
        if (itemType.getTypeTag() != ATypeTag.OBJECT) {
            if (i == 0) {
                return false;
            } else {
                getIxJoinOuterAdditionalDatasets().add(null);
                getIxJoinOuterAdditionalRecordTypes().add(null);
            }
        }
        rType = (ARecordType) itemType;
        // Get the meta record type for that dataset.
        IAType metaItemType = metadataProvider.findType(ds.getMetaItemTypeDataverseName(), ds.getMetaItemTypeName());
        // First index is always the primary datasource in this subtree.
        if (i == 0) {
            setDataset(ds);
            setRecordType(rType);
            setMetaRecordType((ARecordType) metaItemType);
        } else {
            getIxJoinOuterAdditionalDatasets().add(ds);
            getIxJoinOuterAdditionalRecordTypes().add(rType);
        }
        dataverseName = null;
        datasetName = null;
        ds = null;
        rType = null;
    }
    return true;
}