Examples with IMetadataProvider org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider used on opensource projects

Example 6 with IMetadataProvider

use of org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider in project asterixdb by apache.

the class MetadataBuiltinFunctions method addMetadataBuiltinFunctions.

public static void addMetadataBuiltinFunctions() {
    BuiltinFunctions.addFunction(BuiltinFunctions.DATASET, new IResultTypeComputer() {

        @Override
        public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env, IMetadataProvider<?, ?> mp) throws AlgebricksException {
            AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expression;
            if (f.getArguments().size() != 1) {
                throw new AlgebricksException("dataset arity is 1, not " + f.getArguments().size());
            }
            ILogicalExpression a1 = f.getArguments().get(0).getValue();
            IAType t1 = (IAType) env.getType(a1);
            if (t1.getTypeTag() == ATypeTag.ANY) {
                return BuiltinType.ANY;
            }
            if (t1.getTypeTag() != ATypeTag.STRING) {
                throw new AlgebricksException("Illegal type " + t1 + " for dataset() argument.");
            }
            String datasetArg = ConstantExpressionUtil.getStringConstant(a1);
            if (datasetArg == null) {
                return BuiltinType.ANY;
            }
            MetadataProvider metadata = (MetadataProvider) mp;
            Pair<String, String> datasetInfo = getDatasetInfo(metadata, datasetArg);
            String dataverseName = datasetInfo.first;
            String datasetName = datasetInfo.second;
            if (dataverseName == null) {
                throw new AlgebricksException("Unspecified dataverse!");
            }
            Dataset dataset = metadata.findDataset(dataverseName, datasetName);
            if (dataset == null) {
                throw new AlgebricksException("Could not find dataset " + datasetName + " in dataverse " + dataverseName);
            }
            String tn = dataset.getItemTypeName();
            IAType t2 = metadata.findType(dataset.getItemTypeDataverseName(), tn);
            if (t2 == null) {
                throw new AlgebricksException("No type for dataset " + datasetName);
            }
            return t2;
        }
    }, true);
    BuiltinFunctions.addPrivateFunction(BuiltinFunctions.FEED_COLLECT, new IResultTypeComputer() {

        @Override
        public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env, IMetadataProvider<?, ?> mp) throws AlgebricksException {
            AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expression;
            if (f.getArguments().size() != BuiltinFunctions.FEED_COLLECT.getArity()) {
                throw new AlgebricksException("Incorrect number of arguments -> arity is " + BuiltinFunctions.FEED_COLLECT.getArity() + ", not " + f.getArguments().size());
            }
            ILogicalExpression a1 = f.getArguments().get(5).getValue();
            IAType t1 = (IAType) env.getType(a1);
            if (t1.getTypeTag() == ATypeTag.ANY) {
                return BuiltinType.ANY;
            }
            if (t1.getTypeTag() != ATypeTag.STRING) {
                throw new AlgebricksException("Illegal type " + t1 + " for feed-ingest argument.");
            }
            String typeArg = ConstantExpressionUtil.getStringConstant(a1);
            if (typeArg == null) {
                return BuiltinType.ANY;
            }
            MetadataProvider metadata = (MetadataProvider) mp;
            Pair<String, String> argInfo = getDatasetInfo(metadata, typeArg);
            String dataverseName = argInfo.first;
            String typeName = argInfo.second;
            if (dataverseName == null) {
                throw new AlgebricksException("Unspecified dataverse!");
            }
            IAType t2 = metadata.findType(dataverseName, typeName);
            if (t2 == null) {
                throw new AlgebricksException("Unknown type  " + typeName);
            }
            return t2;
        }
    }, true);
    BuiltinFunctions.addFunction(BuiltinFunctions.FEED_INTERCEPT, new IResultTypeComputer() {

        @Override
        public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env, IMetadataProvider<?, ?> mp) throws AlgebricksException {
            AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expression;
            if (f.getArguments().size() != 1) {
                throw new AlgebricksException("dataset arity is 1, not " + f.getArguments().size());
            }
            ILogicalExpression a1 = f.getArguments().get(0).getValue();
            IAType t1 = (IAType) env.getType(a1);
            if (t1.getTypeTag() == ATypeTag.ANY) {
                return BuiltinType.ANY;
            }
            if (t1.getTypeTag() != ATypeTag.STRING) {
                throw new AlgebricksException("Illegal type " + t1 + " for dataset() argument.");
            }
            String datasetArg = ConstantExpressionUtil.getStringConstant(a1);
            if (datasetArg == null) {
                return BuiltinType.ANY;
            }
            MetadataProvider metadata = (MetadataProvider) mp;
            Pair<String, String> datasetInfo = getDatasetInfo(metadata, datasetArg);
            String dataverseName = datasetInfo.first;
            String datasetName = datasetInfo.second;
            if (dataverseName == null) {
                throw new AlgebricksException("Unspecified dataverse!");
            }
            Dataset dataset = metadata.findDataset(dataverseName, datasetName);
            if (dataset == null) {
                throw new AlgebricksException("Could not find dataset " + datasetName + " in dataverse " + dataverseName);
            }
            String tn = dataset.getItemTypeName();
            IAType t2 = metadata.findType(dataset.getItemTypeDataverseName(), tn);
            if (t2 == null) {
                throw new AlgebricksException("No type for dataset " + datasetName);
            }
            return t2;
        }
    }, true);
}

Example 7 with IMetadataProvider

use of org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider in project asterixdb by apache.

the class HeuristicCompilerFactoryBuilder method create.

@Override
public ICompilerFactory create() {
    return new ICompilerFactory() {

        @Override
        public ICompiler createCompiler(final ILogicalPlan plan, final IMetadataProvider<?, ?> metadata, int varCounter) {
            final IOptimizationContext oc = optCtxFactory.createOptimizationContext(varCounter, expressionEvalSizeComputer, mergeAggregationExpressionFactory, expressionTypeComputer, missableTypeComputer, conflictingTypeResolver, physicalOptimizationConfig, clusterLocations);
            oc.setMetadataDeclarations(metadata);
            final HeuristicOptimizer opt = new HeuristicOptimizer(plan, logicalRewrites, physicalRewrites, oc);
            return new ICompiler() {

                @Override
                public void optimize() throws AlgebricksException {
                    opt.optimize();
                }

                @Override
                public JobSpecification createJob(Object appContext, IJobletEventListenerFactory jobEventListenerFactory) throws AlgebricksException {
                    AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Starting Job Generation.\n");
                    JobGenContext context = new JobGenContext(null, metadata, appContext, serializerDeserializerProvider, hashFunctionFactoryProvider, hashFunctionFamilyProvider, comparatorFactoryProvider, typeTraitProvider, binaryBooleanInspectorFactory, binaryIntegerInspectorFactory, printerProvider, missingWriterFactory, normalizedKeyComputerFactoryProvider, expressionRuntimeProvider, expressionTypeComputer, oc, expressionEvalSizeComputer, partialAggregationTypeComputer, predEvaluatorFactoryProvider, physicalOptimizationConfig.getFrameSize(), clusterLocations);
                    PlanCompiler pc = new PlanCompiler(context);
                    return pc.compilePlan(plan, null, jobEventListenerFactory);
                }
            };
        }
    };
}

Example 8 with IMetadataProvider

use of org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider in project asterixdb by apache.

the class DistributeResultPOperator method contributeRuntimeOperator.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
    DistributeResultOperator resultOp = (DistributeResultOperator) op;
    IMetadataProvider mp = context.getMetadataProvider();
    JobSpecification spec = builder.getJobSpec();
    int[] columns = new int[resultOp.getExpressions().size()];
    int i = 0;
    for (Mutable<ILogicalExpression> exprRef : resultOp.getExpressions()) {
        ILogicalExpression expr = exprRef.getValue();
        if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
            throw new NotImplementedException("Only writing variable expressions is supported.");
        }
        VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
        LogicalVariable v = varRef.getVariableReference();
        columns[i++] = inputSchemas[0].findVariable(v);
    }
    RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context);
    IPrinterFactory[] pf = JobGenHelper.mkPrinterFactories(inputSchemas[0], context.getTypeEnvironment(op), context, columns);
    Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getResultHandleRuntime(resultOp.getDataSink(), columns, pf, inputDesc, true, spec);
    builder.contributeHyracksOperator(resultOp, runtimeAndConstraints.first);
    ILogicalOperator src = resultOp.getInputs().get(0).getValue();
    builder.contributeGraphEdge(src, 0, resultOp, 0);
}

Example 9 with IMetadataProvider

use of org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider in project asterixdb by apache.

the class InsertDeleteUpsertPOperator method contributeRuntimeOperator.

@SuppressWarnings("unchecked")
@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
    InsertDeleteUpsertOperator insertDeleteOp = (InsertDeleteUpsertOperator) op;
    IMetadataProvider mp = context.getMetadataProvider();
    IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
    JobSpecification spec = builder.getJobSpec();
    RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context);
    Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = null;
    if (operation == Kind.INSERT) {
        runtimeAndConstraints = mp.getInsertRuntime(dataSource, propagatedSchema, typeEnv, keys, payload, additionalFilteringKeys, additionalNonFilteringFields, inputDesc, context, spec, false);
    } else if (operation == Kind.DELETE) {
        runtimeAndConstraints = mp.getDeleteRuntime(dataSource, propagatedSchema, typeEnv, keys, payload, additionalFilteringKeys, inputDesc, context, spec);
    } else if (operation == Kind.UPSERT) {
        runtimeAndConstraints = mp.getUpsertRuntime(dataSource, inputSchemas[0], typeEnv, keys, payload, additionalFilteringKeys, additionalNonFilteringFields, inputDesc, context, spec);
    } else {
        throw new AlgebricksException("Unsupported Operation " + operation);
    }
    builder.contributeHyracksOperator(insertDeleteOp, runtimeAndConstraints.first);
    builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
    ILogicalOperator src = insertDeleteOp.getInputs().get(0).getValue();
    builder.contributeGraphEdge(src, 0, insertDeleteOp, 0);
}

Example 10 with IMetadataProvider

use of org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider in project asterixdb by apache.

the class IndexInsertDeleteUpsertPOperator method contributeRuntimeOperator.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
    IndexInsertDeleteUpsertOperator insertDeleteUpsertOp = (IndexInsertDeleteUpsertOperator) op;
    IMetadataProvider mp = context.getMetadataProvider();
    JobSpecification spec = builder.getJobSpec();
    RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context);
    Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = null;
    IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(insertDeleteUpsertOp);
    if (insertDeleteUpsertOp.getOperation() == Kind.INSERT) {
        runtimeAndConstraints = mp.getIndexInsertRuntime(dataSourceIndex, propagatedSchema, inputSchemas, typeEnv, primaryKeys, secondaryKeys, additionalFilteringKeys, filterExpr, inputDesc, context, spec, false);
    } else if (insertDeleteUpsertOp.getOperation() == Kind.DELETE) {
        runtimeAndConstraints = mp.getIndexDeleteRuntime(dataSourceIndex, propagatedSchema, inputSchemas, typeEnv, primaryKeys, secondaryKeys, additionalFilteringKeys, filterExpr, inputDesc, context, spec);
    } else if (insertDeleteUpsertOp.getOperation() == Kind.UPSERT) {
        runtimeAndConstraints = mp.getIndexUpsertRuntime(dataSourceIndex, propagatedSchema, inputSchemas, typeEnv, primaryKeys, secondaryKeys, additionalFilteringKeys, filterExpr, prevSecondaryKeys, prevAdditionalFilteringKey, inputDesc, context, spec);
    }
    builder.contributeHyracksOperator(insertDeleteUpsertOp, runtimeAndConstraints.first);
    builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
    ILogicalOperator src = insertDeleteUpsertOp.getInputs().get(0).getValue();
    builder.contributeGraphEdge(src, 0, insertDeleteUpsertOp, 0);
}