Examples with FeedDataSource org.apache.asterix.metadata.declared.FeedDataSource used on opensource projects

Example 1 with FeedDataSource

use of org.apache.asterix.metadata.declared.FeedDataSource in project asterixdb by apache.

the class UnnestToDataScanRule method createFeedDataSource.

private FeedDataSource createFeedDataSource(DataSourceId aqlId, String targetDataset, String sourceFeedName, String subscriptionLocation, MetadataProvider metadataProvider, FeedPolicyEntity feedPolicy, String outputType, String locations, LogicalVariable recordVar, IOptimizationContext context, List<LogicalVariable> pkVars) throws AlgebricksException {
    if (!aqlId.getDataverseName().equals(metadataProvider.getDefaultDataverse() == null ? null : metadataProvider.getDefaultDataverse().getDataverseName())) {
        return null;
    }
    Dataset dataset = metadataProvider.findDataset(aqlId.getDataverseName(), targetDataset);
    ARecordType feedOutputType = (ARecordType) metadataProvider.findType(aqlId.getDataverseName(), outputType);
    Feed sourceFeed = metadataProvider.findFeed(aqlId.getDataverseName(), sourceFeedName);
    FeedConnection feedConnection = metadataProvider.findFeedConnection(aqlId.getDataverseName(), sourceFeedName, targetDataset);
    ARecordType metaType = null;
    // Does dataset have meta?
    if (dataset.hasMetaPart()) {
        String metaTypeName = FeedUtils.getFeedMetaTypeName(sourceFeed.getAdapterConfiguration());
        if (metaTypeName == null) {
            throw new AlgebricksException("Feed to a dataset with metadata doesn't have meta type specified");
        }
        String dataverseName = aqlId.getDataverseName();
        if (metaTypeName.contains(".")) {
            dataverseName = metaTypeName.substring(0, metaTypeName.indexOf('.'));
            metaTypeName = metaTypeName.substring(metaTypeName.indexOf('.') + 1);
        }
        metaType = (ARecordType) metadataProvider.findType(dataverseName, metaTypeName);
    }
    // Is a change feed?
    List<IAType> pkTypes = null;
    List<List<String>> partitioningKeys = null;
    List<Integer> keySourceIndicator = null;
    List<Mutable<ILogicalExpression>> keyAccessExpression = null;
    List<ScalarFunctionCallExpression> keyAccessScalarFunctionCallExpression;
    if (ExternalDataUtils.isChangeFeed(sourceFeed.getAdapterConfiguration())) {
        keyAccessExpression = new ArrayList<>();
        keyAccessScalarFunctionCallExpression = new ArrayList<>();
        pkTypes = ((InternalDatasetDetails) dataset.getDatasetDetails()).getPrimaryKeyType();
        partitioningKeys = ((InternalDatasetDetails) dataset.getDatasetDetails()).getPartitioningKey();
        if (dataset.hasMetaPart()) {
            keySourceIndicator = ((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator();
        }
        for (int i = 0; i < partitioningKeys.size(); i++) {
            List<String> key = partitioningKeys.get(i);
            if (keySourceIndicator == null || keySourceIndicator.get(i).intValue() == 0) {
                PlanTranslationUtil.prepareVarAndExpression(key, recordVar, pkVars, keyAccessExpression, null, context);
            } else {
                PlanTranslationUtil.prepareMetaKeyAccessExpression(key, recordVar, keyAccessExpression, pkVars, null, context);
            }
        }
        keyAccessExpression.forEach(expr -> keyAccessScalarFunctionCallExpression.add((ScalarFunctionCallExpression) expr.getValue()));
    } else {
        keyAccessExpression = null;
        keyAccessScalarFunctionCallExpression = null;
    }
    FeedDataSource feedDataSource = new FeedDataSource(sourceFeed, aqlId, targetDataset, feedOutputType, metaType, pkTypes, partitioningKeys, keyAccessScalarFunctionCallExpression, sourceFeed.getFeedId(), FeedRuntimeType.valueOf(subscriptionLocation), locations.split(","), context.getComputationNodeDomain(), feedConnection);
    feedDataSource.getProperties().put(BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY, feedPolicy);
    return feedDataSource;
}

Example 2 with FeedDataSource

use of org.apache.asterix.metadata.declared.FeedDataSource in project asterixdb by apache.

the class UnnestToDataScanRule method handleFunction.

protected boolean handleFunction(Mutable<ILogicalOperator> opRef, IOptimizationContext context, UnnestOperator unnest, AbstractFunctionCallExpression f) throws AlgebricksException {
    FunctionIdentifier fid = f.getFunctionIdentifier();
    if (fid.equals(BuiltinFunctions.DATASET)) {
        if (unnest.getPositionalVariable() != null) {
            // TODO remove this after enabling the support of positional variables in data scan
            throw new AlgebricksException("No positional variables are allowed over datasets.");
        }
        ILogicalExpression expr = f.getArguments().get(0).getValue();
        if (expr.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
            return false;
        }
        ConstantExpression ce = (ConstantExpression) expr;
        IAlgebricksConstantValue acv = ce.getValue();
        if (!(acv instanceof AsterixConstantValue)) {
            return false;
        }
        AsterixConstantValue acv2 = (AsterixConstantValue) acv;
        if (acv2.getObject().getType().getTypeTag() != ATypeTag.STRING) {
            return false;
        }
        String datasetArg = ((AString) acv2.getObject()).getStringValue();
        MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
        Pair<String, String> datasetReference = parseDatasetReference(metadataProvider, datasetArg);
        String dataverseName = datasetReference.first;
        String datasetName = datasetReference.second;
        Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
        if (dataset == null) {
            throw new AlgebricksException("Could not find dataset " + datasetName + " in dataverse " + dataverseName);
        }
        DataSourceId asid = new DataSourceId(dataverseName, datasetName);
        List<LogicalVariable> variables = new ArrayList<>();
        if (dataset.getDatasetType() == DatasetType.INTERNAL) {
            int numPrimaryKeys = dataset.getPrimaryKeys().size();
            for (int i = 0; i < numPrimaryKeys; i++) {
                variables.add(context.newVar());
            }
        }
        variables.add(unnest.getVariable());
        DataSource dataSource = metadataProvider.findDataSource(asid);
        boolean hasMeta = dataSource.hasMeta();
        if (hasMeta) {
            variables.add(context.newVar());
        }
        DataSourceScanOperator scan = new DataSourceScanOperator(variables, dataSource);
        List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs();
        scanInpList.addAll(unnest.getInputs());
        opRef.setValue(scan);
        addPrimaryKey(variables, dataSource, context);
        context.computeAndSetTypeEnvironmentForOperator(scan);
        // Adds equivalence classes --- one equivalent class between a primary key
        // variable and a record field-access expression.
        IAType[] schemaTypes = dataSource.getSchemaTypes();
        ARecordType recordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 2] : schemaTypes[schemaTypes.length - 1]);
        ARecordType metaRecordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 1] : null);
        EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(scan, variables, recordType, metaRecordType, dataset, context);
        return true;
    } else if (fid.equals(BuiltinFunctions.FEED_COLLECT)) {
        if (unnest.getPositionalVariable() != null) {
            throw new AlgebricksException("No positional variables are allowed over feeds.");
        }
        String dataverse = ConstantExpressionUtil.getStringArgument(f, 0);
        String sourceFeedName = ConstantExpressionUtil.getStringArgument(f, 1);
        String getTargetFeed = ConstantExpressionUtil.getStringArgument(f, 2);
        String subscriptionLocation = ConstantExpressionUtil.getStringArgument(f, 3);
        String targetDataset = ConstantExpressionUtil.getStringArgument(f, 4);
        String outputType = ConstantExpressionUtil.getStringArgument(f, 5);
        MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
        DataSourceId asid = new DataSourceId(dataverse, getTargetFeed);
        String policyName = metadataProvider.getConfig().get(FeedActivityDetails.FEED_POLICY_NAME);
        FeedPolicyEntity policy = metadataProvider.findFeedPolicy(dataverse, policyName);
        if (policy == null) {
            policy = BuiltinFeedPolicies.getFeedPolicy(policyName);
            if (policy == null) {
                throw new AlgebricksException("Unknown feed policy:" + policyName);
            }
        }
        ArrayList<LogicalVariable> feedDataScanOutputVariables = new ArrayList<>();
        String csLocations = metadataProvider.getConfig().get(FeedActivityDetails.COLLECT_LOCATIONS);
        List<LogicalVariable> pkVars = new ArrayList<>();
        FeedDataSource ds = createFeedDataSource(asid, targetDataset, sourceFeedName, subscriptionLocation, metadataProvider, policy, outputType, csLocations, unnest.getVariable(), context, pkVars);
        // The order for feeds is <Record-Meta-PK>
        feedDataScanOutputVariables.add(unnest.getVariable());
        // Does it produce meta?
        if (ds.hasMeta()) {
            feedDataScanOutputVariables.add(context.newVar());
        }
        // Does it produce pk?
        if (ds.isChange()) {
            feedDataScanOutputVariables.addAll(pkVars);
        }
        DataSourceScanOperator scan = new DataSourceScanOperator(feedDataScanOutputVariables, ds);
        List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs();
        scanInpList.addAll(unnest.getInputs());
        opRef.setValue(scan);
        context.computeAndSetTypeEnvironmentForOperator(scan);
        return true;
    }
    return false;
}

Example 3 with FeedDataSource

use of org.apache.asterix.metadata.declared.FeedDataSource in project asterixdb by apache.

the class IntroduceRandomPartitioningFeedComputationRule method rewritePre.

@Override
public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    ILogicalOperator op = opRef.getValue();
    if (!op.getOperatorTag().equals(LogicalOperatorTag.ASSIGN)) {
        return false;
    }
    ILogicalOperator opChild = op.getInputs().get(0).getValue();
    if (!opChild.getOperatorTag().equals(LogicalOperatorTag.DATASOURCESCAN)) {
        return false;
    }
    DataSourceScanOperator scanOp = (DataSourceScanOperator) opChild;
    DataSource dataSource = (DataSource) scanOp.getDataSource();
    if (dataSource.getDatasourceType() != DataSource.Type.FEED) {
        return false;
    }
    final FeedDataSource feedDataSource = (FeedDataSource) dataSource;
    FeedConnection feedConnection = feedDataSource.getFeedConnection();
    if (feedConnection.getAppliedFunctions() == null || feedConnection.getAppliedFunctions().size() == 0) {
        return false;
    }
    ExchangeOperator exchangeOp = new ExchangeOperator();
    INodeDomain domain = new INodeDomain() {

        @Override
        public boolean sameAs(INodeDomain domain) {
            return domain == this;
        }

        @Override
        public Integer cardinality() {
            return feedDataSource.getComputeCardinality();
        }
    };
    exchangeOp.setPhysicalOperator(new RandomPartitionExchangePOperator(domain));
    op.getInputs().get(0).setValue(exchangeOp);
    exchangeOp.getInputs().add(new MutableObject<ILogicalOperator>(scanOp));
    ExecutionMode em = ((AbstractLogicalOperator) scanOp).getExecutionMode();
    exchangeOp.setExecutionMode(em);
    exchangeOp.computeDeliveredPhysicalProperties(context);
    context.computeAndSetTypeEnvironmentForOperator(exchangeOp);
    AssignOperator assignOp = (AssignOperator) opRef.getValue();
    AssignPOperator assignPhyOp = (AssignPOperator) assignOp.getPhysicalOperator();
    assignPhyOp.setCardinalityConstraint(domain.cardinality());
    return true;
}