Examples with ARecordType org.apache.asterix.om.types.ARecordType used on opensource projects

Example 11 with ARecordType

use of org.apache.asterix.om.types.ARecordType in project asterixdb by apache.

the class IntroduceSecondaryIndexInsertDeleteRule method injectFieldAccessesForIndexes.

private ILogicalOperator injectFieldAccessesForIndexes(IOptimizationContext context, Dataset dataset, List<Index> indexes, Map<IndexFieldId, LogicalVariable> fieldAccessVars, ARecordType recType, ARecordType metaType, LogicalVariable recordVar, LogicalVariable metaVar, ILogicalOperator currentTop, boolean afterOp) throws AlgebricksException {
    List<LogicalVariable> vars = new ArrayList<>();
    List<Mutable<ILogicalExpression>> exprs = new ArrayList<>();
    for (Index index : indexes) {
        if (index.isPrimaryIndex()) {
            continue;
        }
        List<IAType> skTypes = index.getKeyFieldTypes();
        List<List<String>> skNames = index.getKeyFieldNames();
        List<Integer> indicators = index.getKeyFieldSourceIndicators();
        for (int i = 0; i < index.getKeyFieldNames().size(); i++) {
            IndexFieldId indexFieldId = new IndexFieldId(indicators.get(i), skNames.get(i));
            if (fieldAccessVars.containsKey(indexFieldId)) {
                // already handled in a different index
                continue;
            }
            ARecordType sourceType = dataset.hasMetaPart() ? indicators.get(i).intValue() == Index.RECORD_INDICATOR ? recType : metaType : recType;
            LogicalVariable sourceVar = dataset.hasMetaPart() ? indicators.get(i).intValue() == Index.RECORD_INDICATOR ? recordVar : metaVar : recordVar;
            LogicalVariable fieldVar = context.newVar();
            // create record variable ref
            Mutable<ILogicalExpression> varRef = new MutableObject<>(new VariableReferenceExpression(sourceVar));
            IAType fieldType = sourceType.getSubFieldType(indexFieldId.fieldName);
            AbstractFunctionCallExpression theFieldAccessFunc;
            if (fieldType == null) {
                // Open field. must prevent inlining to maintain the cast before the primaryOp and
                // make handling of records with incorrect value type for this field easier and cleaner
                context.addNotToBeInlinedVar(fieldVar);
                // create field access
                AbstractFunctionCallExpression fieldAccessFunc = getOpenOrNestedFieldAccessFunction(varRef, indexFieldId.fieldName);
                // create cast
                theFieldAccessFunc = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.CAST_TYPE));
                // The first argument is the field
                theFieldAccessFunc.getArguments().add(new MutableObject<ILogicalExpression>(fieldAccessFunc));
                TypeCastUtils.setRequiredAndInputTypes(theFieldAccessFunc, skTypes.get(i), BuiltinType.ANY);
            } else {
                // Get the desired field position
                int pos = indexFieldId.fieldName.size() > 1 ? -1 : sourceType.getFieldIndex(indexFieldId.fieldName.get(0));
                // Field not found --> This is either an open field or a nested field. it can't be accessed by index
                theFieldAccessFunc = (pos == -1) ? getOpenOrNestedFieldAccessFunction(varRef, indexFieldId.fieldName) : getClosedFieldAccessFunction(varRef, pos);
            }
            vars.add(fieldVar);
            exprs.add(new MutableObject<ILogicalExpression>(theFieldAccessFunc));
            fieldAccessVars.put(indexFieldId, fieldVar);
        }
    }
    // AssignOperator assigns secondary keys to their vars
    AssignOperator castedFieldAssignOperator = new AssignOperator(vars, exprs);
    return introduceNewOp(context, currentTop, castedFieldAssignOperator, afterOp);
}

Example 12 with ARecordType

use of org.apache.asterix.om.types.ARecordType in project asterixdb by apache.

the class PushFieldAccessRule method propagateFieldAccessRec.

@SuppressWarnings("unchecked")
private boolean propagateFieldAccessRec(Mutable<ILogicalOperator> opRef, IOptimizationContext context, String finalAnnot) throws AlgebricksException {
    AssignOperator access = (AssignOperator) opRef.getValue();
    Mutable<ILogicalOperator> opRef2 = access.getInputs().get(0);
    AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
    // rewritten into index search.
    if (op2.getOperatorTag() == LogicalOperatorTag.PROJECT || context.checkAndAddToAlreadyCompared(access, op2) && !(op2.getOperatorTag() == LogicalOperatorTag.SELECT && isAccessToIndexedField(access, context))) {
        return false;
    }
    Object annotation = op2.getAnnotations().get(IS_MOVABLE);
    if (annotation != null && !((Boolean) annotation)) {
        return false;
    }
    if (tryingToPushThroughSelectionWithSameDataSource(access, op2)) {
        return false;
    }
    if (testAndModifyRedundantOp(access, op2)) {
        propagateFieldAccessRec(opRef2, context, finalAnnot);
        return true;
    }
    List<LogicalVariable> usedInAccess = new LinkedList<>();
    VariableUtilities.getUsedVariables(access, usedInAccess);
    List<LogicalVariable> produced2 = new LinkedList<>();
    if (op2.getOperatorTag() == LogicalOperatorTag.GROUP) {
        VariableUtilities.getLiveVariables(op2, produced2);
    } else {
        VariableUtilities.getProducedVariables(op2, produced2);
    }
    boolean pushItDown = false;
    List<LogicalVariable> inter = new ArrayList<>(usedInAccess);
    if (inter.isEmpty()) {
        // ground value
        return false;
    }
    inter.retainAll(produced2);
    if (inter.isEmpty()) {
        pushItDown = true;
    } else if (op2.getOperatorTag() == LogicalOperatorTag.GROUP) {
        GroupByOperator g = (GroupByOperator) op2;
        List<Pair<LogicalVariable, LogicalVariable>> varMappings = new ArrayList<>();
        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : g.getDecorList()) {
            ILogicalExpression e = p.second.getValue();
            if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                LogicalVariable decorVar = GroupByOperator.getDecorVariable(p);
                if (inter.contains(decorVar)) {
                    inter.remove(decorVar);
                    LogicalVariable v1 = ((VariableReferenceExpression) e).getVariableReference();
                    varMappings.add(new Pair<>(decorVar, v1));
                }
            }
        }
        if (inter.isEmpty()) {
            boolean changed = false;
            for (Pair<LogicalVariable, LogicalVariable> m : varMappings) {
                LogicalVariable v2 = context.newVar();
                LogicalVariable oldVar = access.getVariables().get(0);
                g.getDecorList().add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(oldVar, new MutableObject<ILogicalExpression>(new VariableReferenceExpression(v2))));
                changed = true;
                access.getVariables().set(0, v2);
                VariableUtilities.substituteVariables(access, m.first, m.second, context);
            }
            if (changed) {
                context.computeAndSetTypeEnvironmentForOperator(g);
            }
            usedInAccess.clear();
            VariableUtilities.getUsedVariables(access, usedInAccess);
            pushItDown = true;
        }
    }
    if (pushItDown) {
        if (op2.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
            Mutable<ILogicalOperator> childOfSubplan = ((NestedTupleSourceOperator) op2).getDataSourceReference().getValue().getInputs().get(0);
            pushAccessDown(opRef, op2, childOfSubplan, context, finalAnnot);
            return true;
        }
        if (op2.getInputs().size() == 1 && !op2.hasNestedPlans()) {
            pushAccessDown(opRef, op2, op2.getInputs().get(0), context, finalAnnot);
            return true;
        } else {
            for (Mutable<ILogicalOperator> inp : op2.getInputs()) {
                HashSet<LogicalVariable> v2 = new HashSet<>();
                VariableUtilities.getLiveVariables(inp.getValue(), v2);
                if (v2.containsAll(usedInAccess)) {
                    pushAccessDown(opRef, op2, inp, context, finalAnnot);
                    return true;
                }
            }
        }
        if (op2.hasNestedPlans()) {
            AbstractOperatorWithNestedPlans nestedOp = (AbstractOperatorWithNestedPlans) op2;
            for (ILogicalPlan plan : nestedOp.getNestedPlans()) {
                for (Mutable<ILogicalOperator> root : plan.getRoots()) {
                    HashSet<LogicalVariable> v2 = new HashSet<>();
                    VariableUtilities.getLiveVariables(root.getValue(), v2);
                    if (v2.containsAll(usedInAccess)) {
                        pushAccessDown(opRef, op2, root, context, finalAnnot);
                        return true;
                    }
                }
            }
        }
        throw new AlgebricksException("Field access " + access.getExpressions().get(0).getValue() + " does not correspond to any input of operator " + op2);
    } else {
        // fields. If yes, we can equate the two variables.
        if (op2.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
            DataSourceScanOperator scan = (DataSourceScanOperator) op2;
            int n = scan.getVariables().size();
            LogicalVariable scanRecordVar = scan.getVariables().get(n - 1);
            AbstractFunctionCallExpression accessFun = (AbstractFunctionCallExpression) access.getExpressions().get(0).getValue();
            ILogicalExpression e0 = accessFun.getArguments().get(0).getValue();
            LogicalExpressionTag tag = e0.getExpressionTag();
            if (tag == LogicalExpressionTag.VARIABLE) {
                VariableReferenceExpression varRef = (VariableReferenceExpression) e0;
                if (varRef.getVariableReference() == scanRecordVar) {
                    ILogicalExpression e1 = accessFun.getArguments().get(1).getValue();
                    if (e1.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
                        IDataSource<DataSourceId> dataSource = (IDataSource<DataSourceId>) scan.getDataSource();
                        byte dsType = ((DataSource) dataSource).getDatasourceType();
                        if (dsType == DataSource.Type.FEED || dsType == DataSource.Type.LOADABLE) {
                            return false;
                        }
                        DataSourceId asid = dataSource.getId();
                        MetadataProvider mp = (MetadataProvider) context.getMetadataProvider();
                        Dataset dataset = mp.findDataset(asid.getDataverseName(), asid.getDatasourceName());
                        if (dataset == null) {
                            throw new AlgebricksException("Dataset " + asid.getDatasourceName() + " not found.");
                        }
                        if (dataset.getDatasetType() != DatasetType.INTERNAL) {
                            setAsFinal(access, context, finalAnnot);
                            return false;
                        }
                        ConstantExpression ce = (ConstantExpression) e1;
                        IAObject obj = ((AsterixConstantValue) ce.getValue()).getObject();
                        String fldName;
                        if (obj.getType().getTypeTag() == ATypeTag.STRING) {
                            fldName = ((AString) obj).getStringValue();
                        } else {
                            int pos = ((AInt32) obj).getIntegerValue();
                            String tName = dataset.getItemTypeName();
                            IAType t = mp.findType(dataset.getItemTypeDataverseName(), tName);
                            if (t.getTypeTag() != ATypeTag.OBJECT) {
                                return false;
                            }
                            ARecordType rt = (ARecordType) t;
                            if (pos >= rt.getFieldNames().length) {
                                setAsFinal(access, context, finalAnnot);
                                return false;
                            }
                            fldName = rt.getFieldNames()[pos];
                        }
                        int p = DatasetUtil.getPositionOfPartitioningKeyField(dataset, fldName);
                        if (p < 0) {
                            // not one of the partitioning fields
                            setAsFinal(access, context, finalAnnot);
                            return false;
                        }
                        LogicalVariable keyVar = scan.getVariables().get(p);
                        access.getExpressions().get(0).setValue(new VariableReferenceExpression(keyVar));
                        return true;
                    }
                }
            }
        }
        setAsFinal(access, context, finalAnnot);
        return false;
    }
}

Example 13 with ARecordType

use of org.apache.asterix.om.types.ARecordType in project asterixdb by apache.

the class UnnestToDataScanRule method createFeedDataSource.

private FeedDataSource createFeedDataSource(DataSourceId aqlId, String targetDataset, String sourceFeedName, String subscriptionLocation, MetadataProvider metadataProvider, FeedPolicyEntity feedPolicy, String outputType, String locations, LogicalVariable recordVar, IOptimizationContext context, List<LogicalVariable> pkVars) throws AlgebricksException {
    if (!aqlId.getDataverseName().equals(metadataProvider.getDefaultDataverse() == null ? null : metadataProvider.getDefaultDataverse().getDataverseName())) {
        return null;
    }
    Dataset dataset = metadataProvider.findDataset(aqlId.getDataverseName(), targetDataset);
    ARecordType feedOutputType = (ARecordType) metadataProvider.findType(aqlId.getDataverseName(), outputType);
    Feed sourceFeed = metadataProvider.findFeed(aqlId.getDataverseName(), sourceFeedName);
    FeedConnection feedConnection = metadataProvider.findFeedConnection(aqlId.getDataverseName(), sourceFeedName, targetDataset);
    ARecordType metaType = null;
    // Does dataset have meta?
    if (dataset.hasMetaPart()) {
        String metaTypeName = FeedUtils.getFeedMetaTypeName(sourceFeed.getAdapterConfiguration());
        if (metaTypeName == null) {
            throw new AlgebricksException("Feed to a dataset with metadata doesn't have meta type specified");
        }
        String dataverseName = aqlId.getDataverseName();
        if (metaTypeName.contains(".")) {
            dataverseName = metaTypeName.substring(0, metaTypeName.indexOf('.'));
            metaTypeName = metaTypeName.substring(metaTypeName.indexOf('.') + 1);
        }
        metaType = (ARecordType) metadataProvider.findType(dataverseName, metaTypeName);
    }
    // Is a change feed?
    List<IAType> pkTypes = null;
    List<List<String>> partitioningKeys = null;
    List<Integer> keySourceIndicator = null;
    List<Mutable<ILogicalExpression>> keyAccessExpression = null;
    List<ScalarFunctionCallExpression> keyAccessScalarFunctionCallExpression;
    if (ExternalDataUtils.isChangeFeed(sourceFeed.getAdapterConfiguration())) {
        keyAccessExpression = new ArrayList<>();
        keyAccessScalarFunctionCallExpression = new ArrayList<>();
        pkTypes = ((InternalDatasetDetails) dataset.getDatasetDetails()).getPrimaryKeyType();
        partitioningKeys = ((InternalDatasetDetails) dataset.getDatasetDetails()).getPartitioningKey();
        if (dataset.hasMetaPart()) {
            keySourceIndicator = ((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator();
        }
        for (int i = 0; i < partitioningKeys.size(); i++) {
            List<String> key = partitioningKeys.get(i);
            if (keySourceIndicator == null || keySourceIndicator.get(i).intValue() == 0) {
                PlanTranslationUtil.prepareVarAndExpression(key, recordVar, pkVars, keyAccessExpression, null, context);
            } else {
                PlanTranslationUtil.prepareMetaKeyAccessExpression(key, recordVar, keyAccessExpression, pkVars, null, context);
            }
        }
        keyAccessExpression.forEach(expr -> keyAccessScalarFunctionCallExpression.add((ScalarFunctionCallExpression) expr.getValue()));
    } else {
        keyAccessExpression = null;
        keyAccessScalarFunctionCallExpression = null;
    }
    FeedDataSource feedDataSource = new FeedDataSource(sourceFeed, aqlId, targetDataset, feedOutputType, metaType, pkTypes, partitioningKeys, keyAccessScalarFunctionCallExpression, sourceFeed.getFeedId(), FeedRuntimeType.valueOf(subscriptionLocation), locations.split(","), context.getComputationNodeDomain(), feedConnection);
    feedDataSource.getProperties().put(BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY, feedPolicy);
    return feedDataSource;
}

Example 14 with ARecordType

use of org.apache.asterix.om.types.ARecordType in project asterixdb by apache.

the class UnnestToDataScanRule method handleFunction.

protected boolean handleFunction(Mutable<ILogicalOperator> opRef, IOptimizationContext context, UnnestOperator unnest, AbstractFunctionCallExpression f) throws AlgebricksException {
    FunctionIdentifier fid = f.getFunctionIdentifier();
    if (fid.equals(BuiltinFunctions.DATASET)) {
        if (unnest.getPositionalVariable() != null) {
            // TODO remove this after enabling the support of positional variables in data scan
            throw new AlgebricksException("No positional variables are allowed over datasets.");
        }
        ILogicalExpression expr = f.getArguments().get(0).getValue();
        if (expr.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
            return false;
        }
        ConstantExpression ce = (ConstantExpression) expr;
        IAlgebricksConstantValue acv = ce.getValue();
        if (!(acv instanceof AsterixConstantValue)) {
            return false;
        }
        AsterixConstantValue acv2 = (AsterixConstantValue) acv;
        if (acv2.getObject().getType().getTypeTag() != ATypeTag.STRING) {
            return false;
        }
        String datasetArg = ((AString) acv2.getObject()).getStringValue();
        MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
        Pair<String, String> datasetReference = parseDatasetReference(metadataProvider, datasetArg);
        String dataverseName = datasetReference.first;
        String datasetName = datasetReference.second;
        Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
        if (dataset == null) {
            throw new AlgebricksException("Could not find dataset " + datasetName + " in dataverse " + dataverseName);
        }
        DataSourceId asid = new DataSourceId(dataverseName, datasetName);
        List<LogicalVariable> variables = new ArrayList<>();
        if (dataset.getDatasetType() == DatasetType.INTERNAL) {
            int numPrimaryKeys = dataset.getPrimaryKeys().size();
            for (int i = 0; i < numPrimaryKeys; i++) {
                variables.add(context.newVar());
            }
        }
        variables.add(unnest.getVariable());
        DataSource dataSource = metadataProvider.findDataSource(asid);
        boolean hasMeta = dataSource.hasMeta();
        if (hasMeta) {
            variables.add(context.newVar());
        }
        DataSourceScanOperator scan = new DataSourceScanOperator(variables, dataSource);
        List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs();
        scanInpList.addAll(unnest.getInputs());
        opRef.setValue(scan);
        addPrimaryKey(variables, dataSource, context);
        context.computeAndSetTypeEnvironmentForOperator(scan);
        // Adds equivalence classes --- one equivalent class between a primary key
        // variable and a record field-access expression.
        IAType[] schemaTypes = dataSource.getSchemaTypes();
        ARecordType recordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 2] : schemaTypes[schemaTypes.length - 1]);
        ARecordType metaRecordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 1] : null);
        EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(scan, variables, recordType, metaRecordType, dataset, context);
        return true;
    } else if (fid.equals(BuiltinFunctions.FEED_COLLECT)) {
        if (unnest.getPositionalVariable() != null) {
            throw new AlgebricksException("No positional variables are allowed over feeds.");
        }
        String dataverse = ConstantExpressionUtil.getStringArgument(f, 0);
        String sourceFeedName = ConstantExpressionUtil.getStringArgument(f, 1);
        String getTargetFeed = ConstantExpressionUtil.getStringArgument(f, 2);
        String subscriptionLocation = ConstantExpressionUtil.getStringArgument(f, 3);
        String targetDataset = ConstantExpressionUtil.getStringArgument(f, 4);
        String outputType = ConstantExpressionUtil.getStringArgument(f, 5);
        MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
        DataSourceId asid = new DataSourceId(dataverse, getTargetFeed);
        String policyName = metadataProvider.getConfig().get(FeedActivityDetails.FEED_POLICY_NAME);
        FeedPolicyEntity policy = metadataProvider.findFeedPolicy(dataverse, policyName);
        if (policy == null) {
            policy = BuiltinFeedPolicies.getFeedPolicy(policyName);
            if (policy == null) {
                throw new AlgebricksException("Unknown feed policy:" + policyName);
            }
        }
        ArrayList<LogicalVariable> feedDataScanOutputVariables = new ArrayList<>();
        String csLocations = metadataProvider.getConfig().get(FeedActivityDetails.COLLECT_LOCATIONS);
        List<LogicalVariable> pkVars = new ArrayList<>();
        FeedDataSource ds = createFeedDataSource(asid, targetDataset, sourceFeedName, subscriptionLocation, metadataProvider, policy, outputType, csLocations, unnest.getVariable(), context, pkVars);
        // The order for feeds is <Record-Meta-PK>
        feedDataScanOutputVariables.add(unnest.getVariable());
        // Does it produce meta?
        if (ds.hasMeta()) {
            feedDataScanOutputVariables.add(context.newVar());
        }
        // Does it produce pk?
        if (ds.isChange()) {
            feedDataScanOutputVariables.addAll(pkVars);
        }
        DataSourceScanOperator scan = new DataSourceScanOperator(feedDataScanOutputVariables, ds);
        List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs();
        scanInpList.addAll(unnest.getInputs());
        opRef.setValue(scan);
        context.computeAndSetTypeEnvironmentForOperator(scan);
        return true;
    }
    return false;
}

Example 15 with ARecordType

use of org.apache.asterix.om.types.ARecordType in project asterixdb by apache.

the class JSONDeserializerForTypes method convertFromJSON.

/**
     * Deserialize an arbitrary JSON representation of a type.
     *
     * @param typeInJSON
     *            the JSON representation of the type.
     * @return an valid AsterixDB type.
     * @throws Exception
     */
public static IAType convertFromJSON(JsonNode typeInJSON) throws Exception {
    String typeName = typeInJSON.get("type").asText();
    // Deals with ordered list.
    if (typeName.equals(AOrderedListType.class.getName())) {
        IAType itemType = convertFromJSON(typeInJSON.get("item-type"));
        return new AOrderedListType(itemType, "ordered-list");
    }
    // Deals with unordered list.
    if (typeName.equals(AUnorderedListType.class.getName())) {
        IAType itemType = convertFromJSON(typeInJSON.get("item-type"));
        return new AUnorderedListType(itemType, "unordered-list");
    }
    // Deals with Union Type.
    if (typeName.equals(AUnionType.class.getName())) {
        List<IAType> unionTypes = new ArrayList<IAType>();
        JsonNode fields = typeInJSON.get("fields");
        for (int i = 0; i < fields.size(); i++) {
            JsonNode fieldType = fields.get(i);
            unionTypes.add(convertFromJSON(fieldType));
        }
        return new AUnionType(unionTypes, "union");
    }
    // Deals with record types.
    if (typeName.equals(ARecordType.class.getName())) {
        String name = typeInJSON.get("name").asText();
        boolean openType = typeInJSON.get("open").asBoolean();
        JsonNode fields = typeInJSON.get("fields");
        String[] fieldNames = new String[fields.size()];
        IAType[] fieldTypes = new IAType[fields.size()];
        for (int i = 0; i < fields.size(); ++i) {
            JsonNode field = fields.get(i);
            List<String> names = Lists.newArrayList(field.fieldNames());
            String fieldName = names.get(0);
            fieldNames[i] = fieldName;
            fieldTypes[i] = convertFromJSON(field.get(fieldName));
        }
        return new ARecordType(name, fieldNames, fieldTypes, openType);
    }
    // Deals with primitive types.
    Class<?> cl = BuiltinType.class;
    Field typeField = cl.getDeclaredField(typeName.toUpperCase());
    return (IAType) typeField.get(null);
}