Examples with Dataset org.apache.asterix.metadata.entities.Dataset used on opensource projects

Example 66 with Dataset

use of org.apache.asterix.metadata.entities.Dataset in project asterixdb by apache.

the class MetadataBuiltinFunctions method addMetadataBuiltinFunctions.

public static void addMetadataBuiltinFunctions() {
    BuiltinFunctions.addFunction(BuiltinFunctions.DATASET, new IResultTypeComputer() {

        @Override
        public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env, IMetadataProvider<?, ?> mp) throws AlgebricksException {
            AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expression;
            if (f.getArguments().size() != 1) {
                throw new AlgebricksException("dataset arity is 1, not " + f.getArguments().size());
            }
            ILogicalExpression a1 = f.getArguments().get(0).getValue();
            IAType t1 = (IAType) env.getType(a1);
            if (t1.getTypeTag() == ATypeTag.ANY) {
                return BuiltinType.ANY;
            }
            if (t1.getTypeTag() != ATypeTag.STRING) {
                throw new AlgebricksException("Illegal type " + t1 + " for dataset() argument.");
            }
            String datasetArg = ConstantExpressionUtil.getStringConstant(a1);
            if (datasetArg == null) {
                return BuiltinType.ANY;
            }
            MetadataProvider metadata = (MetadataProvider) mp;
            Pair<String, String> datasetInfo = getDatasetInfo(metadata, datasetArg);
            String dataverseName = datasetInfo.first;
            String datasetName = datasetInfo.second;
            if (dataverseName == null) {
                throw new AlgebricksException("Unspecified dataverse!");
            }
            Dataset dataset = metadata.findDataset(dataverseName, datasetName);
            if (dataset == null) {
                throw new AlgebricksException("Could not find dataset " + datasetName + " in dataverse " + dataverseName);
            }
            String tn = dataset.getItemTypeName();
            IAType t2 = metadata.findType(dataset.getItemTypeDataverseName(), tn);
            if (t2 == null) {
                throw new AlgebricksException("No type for dataset " + datasetName);
            }
            return t2;
        }
    }, true);
    BuiltinFunctions.addPrivateFunction(BuiltinFunctions.FEED_COLLECT, new IResultTypeComputer() {

        @Override
        public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env, IMetadataProvider<?, ?> mp) throws AlgebricksException {
            AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expression;
            if (f.getArguments().size() != BuiltinFunctions.FEED_COLLECT.getArity()) {
                throw new AlgebricksException("Incorrect number of arguments -> arity is " + BuiltinFunctions.FEED_COLLECT.getArity() + ", not " + f.getArguments().size());
            }
            ILogicalExpression a1 = f.getArguments().get(5).getValue();
            IAType t1 = (IAType) env.getType(a1);
            if (t1.getTypeTag() == ATypeTag.ANY) {
                return BuiltinType.ANY;
            }
            if (t1.getTypeTag() != ATypeTag.STRING) {
                throw new AlgebricksException("Illegal type " + t1 + " for feed-ingest argument.");
            }
            String typeArg = ConstantExpressionUtil.getStringConstant(a1);
            if (typeArg == null) {
                return BuiltinType.ANY;
            }
            MetadataProvider metadata = (MetadataProvider) mp;
            Pair<String, String> argInfo = getDatasetInfo(metadata, typeArg);
            String dataverseName = argInfo.first;
            String typeName = argInfo.second;
            if (dataverseName == null) {
                throw new AlgebricksException("Unspecified dataverse!");
            }
            IAType t2 = metadata.findType(dataverseName, typeName);
            if (t2 == null) {
                throw new AlgebricksException("Unknown type  " + typeName);
            }
            return t2;
        }
    }, true);
    BuiltinFunctions.addFunction(BuiltinFunctions.FEED_INTERCEPT, new IResultTypeComputer() {

        @Override
        public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env, IMetadataProvider<?, ?> mp) throws AlgebricksException {
            AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expression;
            if (f.getArguments().size() != 1) {
                throw new AlgebricksException("dataset arity is 1, not " + f.getArguments().size());
            }
            ILogicalExpression a1 = f.getArguments().get(0).getValue();
            IAType t1 = (IAType) env.getType(a1);
            if (t1.getTypeTag() == ATypeTag.ANY) {
                return BuiltinType.ANY;
            }
            if (t1.getTypeTag() != ATypeTag.STRING) {
                throw new AlgebricksException("Illegal type " + t1 + " for dataset() argument.");
            }
            String datasetArg = ConstantExpressionUtil.getStringConstant(a1);
            if (datasetArg == null) {
                return BuiltinType.ANY;
            }
            MetadataProvider metadata = (MetadataProvider) mp;
            Pair<String, String> datasetInfo = getDatasetInfo(metadata, datasetArg);
            String dataverseName = datasetInfo.first;
            String datasetName = datasetInfo.second;
            if (dataverseName == null) {
                throw new AlgebricksException("Unspecified dataverse!");
            }
            Dataset dataset = metadata.findDataset(dataverseName, datasetName);
            if (dataset == null) {
                throw new AlgebricksException("Could not find dataset " + datasetName + " in dataverse " + dataverseName);
            }
            String tn = dataset.getItemTypeName();
            IAType t2 = metadata.findType(dataset.getItemTypeDataverseName(), tn);
            if (t2 == null) {
                throw new AlgebricksException("No type for dataset " + datasetName);
            }
            return t2;
        }
    }, true);
}

Example 67 with Dataset

use of org.apache.asterix.metadata.entities.Dataset in project asterixdb by apache.

the class DatasetUtil method compactDatasetJobSpec.

public static JobSpecification compactDatasetJobSpec(Dataverse dataverse, String datasetName, MetadataProvider metadataProvider) throws AlgebricksException {
    String dataverseName = dataverse.getDataverseName();
    Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
    if (dataset == null) {
        throw new AsterixException("Could not find dataset " + datasetName + " in dataverse " + dataverseName);
    }
    JobSpecification spec = RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext());
    Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider.getSplitProviderAndConstraints(dataset);
    IIndexDataflowHelperFactory indexHelperFactory = new IndexDataflowHelperFactory(metadataProvider.getStorageComponentProvider().getStorageManager(), splitsAndConstraint.first);
    LSMTreeIndexCompactOperatorDescriptor compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec, indexHelperFactory);
    AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp, splitsAndConstraint.second);
    AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp, splitsAndConstraint.second);
    spec.addRoot(compactOp);
    return spec;
}

Example 68 with Dataset

use of org.apache.asterix.metadata.entities.Dataset in project asterixdb by apache.

the class InvertedIndexPOperator method contributeRuntimeOperator.

@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
    AbstractUnnestMapOperator unnestMapOp = (AbstractUnnestMapOperator) op;
    ILogicalExpression unnestExpr = unnestMapOp.getExpressionRef().getValue();
    if (unnestExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
        throw new IllegalStateException();
    }
    AbstractFunctionCallExpression unnestFuncExpr = (AbstractFunctionCallExpression) unnestExpr;
    if (unnestFuncExpr.getFunctionIdentifier() != BuiltinFunctions.INDEX_SEARCH) {
        return;
    }
    InvertedIndexJobGenParams jobGenParams = new InvertedIndexJobGenParams();
    jobGenParams.readFromFuncArgs(unnestFuncExpr.getArguments());
    MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
    Dataset dataset;
    try {
        dataset = metadataProvider.findDataset(jobGenParams.getDataverseName(), jobGenParams.getDatasetName());
    } catch (MetadataException e) {
        throw new AlgebricksException(e);
    }
    int[] keyIndexes = getKeyIndexes(jobGenParams.getKeyVarList(), inputSchemas);
    int[] minFilterFieldIndexes = getKeyIndexes(unnestMapOp.getMinFilterVars(), inputSchemas);
    int[] maxFilterFieldIndexes = getKeyIndexes(unnestMapOp.getMaxFilterVars(), inputSchemas);
    boolean retainNull = false;
    if (op.getOperatorTag() == LogicalOperatorTag.LEFT_OUTER_UNNEST_MAP) {
        // By nature, LEFT_OUTER_UNNEST_MAP should generate null values for non-matching tuples.
        retainNull = true;
    }
    // Build runtime.
    Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> invIndexSearch = buildInvertedIndexRuntime(metadataProvider, context, builder.getJobSpec(), unnestMapOp, opSchema, jobGenParams.getRetainInput(), retainNull, jobGenParams.getDatasetName(), dataset, jobGenParams.getIndexName(), jobGenParams.getSearchKeyType(), keyIndexes, jobGenParams.getSearchModifierType(), jobGenParams.getSimilarityThreshold(), minFilterFieldIndexes, maxFilterFieldIndexes, jobGenParams.getIsFullTextSearch());
    // Contribute operator in hyracks job.
    builder.contributeHyracksOperator(unnestMapOp, invIndexSearch.first);
    builder.contributeAlgebricksPartitionConstraint(invIndexSearch.first, invIndexSearch.second);
    ILogicalOperator srcExchange = unnestMapOp.getInputs().get(0).getValue();
    builder.contributeGraphEdge(srcExchange, 0, unnestMapOp, 0);
}

Example 69 with Dataset

use of org.apache.asterix.metadata.entities.Dataset in project asterixdb by apache.

the class RTreeSearchPOperator method contributeRuntimeOperator.

@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
    AbstractUnnestMapOperator unnestMap = (AbstractUnnestMapOperator) op;
    ILogicalExpression unnestExpr = unnestMap.getExpressionRef().getValue();
    if (unnestExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
        throw new IllegalStateException();
    }
    AbstractFunctionCallExpression unnestFuncExpr = (AbstractFunctionCallExpression) unnestExpr;
    FunctionIdentifier funcIdent = unnestFuncExpr.getFunctionIdentifier();
    if (!funcIdent.equals(BuiltinFunctions.INDEX_SEARCH)) {
        return;
    }
    RTreeJobGenParams jobGenParams = new RTreeJobGenParams();
    jobGenParams.readFromFuncArgs(unnestFuncExpr.getArguments());
    int[] keyIndexes = getKeyIndexes(jobGenParams.getKeyVarList(), inputSchemas);
    int[] minFilterFieldIndexes = getKeyIndexes(unnestMap.getMinFilterVars(), inputSchemas);
    int[] maxFilterFieldIndexes = getKeyIndexes(unnestMap.getMaxFilterVars(), inputSchemas);
    MetadataProvider mp = (MetadataProvider) context.getMetadataProvider();
    Dataset dataset = mp.findDataset(jobGenParams.getDataverseName(), jobGenParams.getDatasetName());
    IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(unnestMap);
    List<LogicalVariable> outputVars = unnestMap.getVariables();
    if (jobGenParams.getRetainInput()) {
        outputVars = new ArrayList<LogicalVariable>();
        VariableUtilities.getLiveVariables(unnestMap, outputVars);
    }
    boolean retainNull = false;
    if (op.getOperatorTag() == LogicalOperatorTag.LEFT_OUTER_UNNEST_MAP) {
        // By nature, LEFT_OUTER_UNNEST_MAP should generate null values for non-matching tuples.
        retainNull = true;
    }
    Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> rtreeSearch = mp.buildRtreeRuntime(builder.getJobSpec(), outputVars, opSchema, typeEnv, context, jobGenParams.getRetainInput(), retainNull, dataset, jobGenParams.getIndexName(), keyIndexes, minFilterFieldIndexes, maxFilterFieldIndexes);
    builder.contributeHyracksOperator(unnestMap, rtreeSearch.first);
    builder.contributeAlgebricksPartitionConstraint(rtreeSearch.first, rtreeSearch.second);
    ILogicalOperator srcExchange = unnestMap.getInputs().get(0).getValue();
    builder.contributeGraphEdge(srcExchange, 0, unnestMap, 0);
}

Example 70 with Dataset

use of org.apache.asterix.metadata.entities.Dataset in project asterixdb by apache.

the class BTreeSearchPOperator method contributeRuntimeOperator.

@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
    AbstractUnnestMapOperator unnestMap = (AbstractUnnestMapOperator) op;
    ILogicalExpression unnestExpr = unnestMap.getExpressionRef().getValue();
    if (unnestExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
        throw new IllegalStateException();
    }
    AbstractFunctionCallExpression unnestFuncExpr = (AbstractFunctionCallExpression) unnestExpr;
    FunctionIdentifier funcIdent = unnestFuncExpr.getFunctionIdentifier();
    if (!funcIdent.equals(BuiltinFunctions.INDEX_SEARCH)) {
        return;
    }
    BTreeJobGenParams jobGenParams = new BTreeJobGenParams();
    jobGenParams.readFromFuncArgs(unnestFuncExpr.getArguments());
    int[] lowKeyIndexes = getKeyIndexes(jobGenParams.getLowKeyVarList(), inputSchemas);
    int[] highKeyIndexes = getKeyIndexes(jobGenParams.getHighKeyVarList(), inputSchemas);
    int[] minFilterFieldIndexes = getKeyIndexes(unnestMap.getMinFilterVars(), inputSchemas);
    int[] maxFilterFieldIndexes = getKeyIndexes(unnestMap.getMaxFilterVars(), inputSchemas);
    MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
    Dataset dataset = metadataProvider.findDataset(jobGenParams.getDataverseName(), jobGenParams.getDatasetName());
    IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
    // By nature, LEFT_OUTER_UNNEST_MAP should generate null values for non-matching tuples.
    boolean retainMissing = op.getOperatorTag() == LogicalOperatorTag.LEFT_OUTER_UNNEST_MAP;
    Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> btreeSearch = metadataProvider.buildBtreeRuntime(builder.getJobSpec(), opSchema, typeEnv, context, jobGenParams.getRetainInput(), retainMissing, dataset, jobGenParams.getIndexName(), lowKeyIndexes, highKeyIndexes, jobGenParams.isLowKeyInclusive(), jobGenParams.isHighKeyInclusive(), minFilterFieldIndexes, maxFilterFieldIndexes);
    builder.contributeHyracksOperator(unnestMap, btreeSearch.first);
    builder.contributeAlgebricksPartitionConstraint(btreeSearch.first, btreeSearch.second);
    ILogicalOperator srcExchange = unnestMap.getInputs().get(0).getValue();
    builder.contributeGraphEdge(srcExchange, 0, unnestMap, 0);
}