Search in sources :

Example 21 with AlgebricksPartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint in project asterixdb by apache.

the class JobBuilder method contributeMicroOperator.

@Override
public void contributeMicroOperator(ILogicalOperator op, IPushRuntimeFactory runtime, RecordDescriptor recDesc, AlgebricksPartitionConstraint pc) {
    microOps.put(op, new Pair<IPushRuntimeFactory, RecordDescriptor>(runtime, recDesc));
    revMicroOpMap.put(runtime, op);
    if (pc != null) {
        pcForMicroOps.put(op, pc);
    }
    AbstractLogicalOperator logicalOp = (AbstractLogicalOperator) op;
    if (logicalOp.getExecutionMode() == ExecutionMode.UNPARTITIONED && pc == null) {
        AlgebricksPartitionConstraint apc = countOneLocation;
        pcForMicroOps.put(logicalOp, apc);
    }
}
Also used : AbstractLogicalOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator) RecordDescriptor(org.apache.hyracks.api.dataflow.value.RecordDescriptor) AlgebricksPartitionConstraint(org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint) IPushRuntimeFactory(org.apache.hyracks.algebricks.runtime.base.IPushRuntimeFactory)

Example 22 with AlgebricksPartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint in project asterixdb by apache.

the class BulkloadPOperator method contributeRuntimeOperator.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
    InsertDeleteUpsertOperator insertDeleteOp = (InsertDeleteUpsertOperator) op;
    assert insertDeleteOp.getOperation() == Kind.INSERT;
    assert insertDeleteOp.isBulkload();
    IMetadataProvider mp = context.getMetadataProvider();
    IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
    JobSpecification spec = builder.getJobSpec();
    RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context);
    Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getInsertRuntime(dataSource, propagatedSchema, typeEnv, primaryKeys, payload, additionalFilteringKeys, additionalNonFilterVars, inputDesc, context, spec, true);
    builder.contributeHyracksOperator(insertDeleteOp, runtimeAndConstraints.first);
    builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
    ILogicalOperator src = insertDeleteOp.getInputs().get(0).getValue();
    builder.contributeGraphEdge(src, 0, insertDeleteOp, 0);
}
Also used : InsertDeleteUpsertOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator) RecordDescriptor(org.apache.hyracks.api.dataflow.value.RecordDescriptor) IOperatorDescriptor(org.apache.hyracks.api.dataflow.IOperatorDescriptor) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator) AlgebricksPartitionConstraint(org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint) JobSpecification(org.apache.hyracks.api.job.JobSpecification) IMetadataProvider(org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider) IVariableTypeEnvironment(org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment)

Example 23 with AlgebricksPartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint in project asterixdb by apache.

the class DataSourceScanPOperator method contributeRuntimeOperator.

@SuppressWarnings("unchecked")
@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
    DataSourceScanOperator scan = (DataSourceScanOperator) op;
    IMetadataProvider mp = context.getMetadataProvider();
    IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
    List<LogicalVariable> vars = scan.getVariables();
    List<LogicalVariable> projectVars = scan.getProjectVariables();
    Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> p = mp.getScannerRuntime(dataSource, vars, projectVars, scan.isProjectPushed(), scan.getMinFilterVars(), scan.getMaxFilterVars(), opSchema, typeEnv, context, builder.getJobSpec(), implConfig);
    builder.contributeHyracksOperator(scan, p.first);
    if (p.second != null) {
        builder.contributeAlgebricksPartitionConstraint(p.first, p.second);
    }
    ILogicalOperator srcExchange = scan.getInputs().get(0).getValue();
    builder.contributeGraphEdge(srcExchange, 0, scan, 0);
}
Also used : LogicalVariable(org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable) DataSourceScanOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator) IOperatorDescriptor(org.apache.hyracks.api.dataflow.IOperatorDescriptor) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator) AlgebricksPartitionConstraint(org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint) IMetadataProvider(org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider) IVariableTypeEnvironment(org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment)

Example 24 with AlgebricksPartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint in project asterixdb by apache.

the class IndexBulkloadPOperator method contributeRuntimeOperator.

@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
    IndexInsertDeleteUpsertOperator indexInsertDeleteOp = (IndexInsertDeleteUpsertOperator) op;
    assert indexInsertDeleteOp.getOperation() == Kind.INSERT;
    assert indexInsertDeleteOp.isBulkload();
    IMetadataProvider mp = context.getMetadataProvider();
    IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
    JobSpecification spec = builder.getJobSpec();
    RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context);
    Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getIndexInsertRuntime(dataSourceIndex, propagatedSchema, inputSchemas, typeEnv, primaryKeys, secondaryKeys, additionalFilteringKeys, filterExpr, inputDesc, context, spec, true);
    builder.contributeHyracksOperator(indexInsertDeleteOp, runtimeAndConstraints.first);
    builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
    ILogicalOperator src = indexInsertDeleteOp.getInputs().get(0).getValue();
    builder.contributeGraphEdge(src, 0, indexInsertDeleteOp, 0);
}
Also used : RecordDescriptor(org.apache.hyracks.api.dataflow.value.RecordDescriptor) IOperatorDescriptor(org.apache.hyracks.api.dataflow.IOperatorDescriptor) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator) IndexInsertDeleteUpsertOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator) AlgebricksPartitionConstraint(org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint) JobSpecification(org.apache.hyracks.api.job.JobSpecification) IMetadataProvider(org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider) IVariableTypeEnvironment(org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment)

Example 25 with AlgebricksPartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint in project asterixdb by apache.

the class InvertedIndexPOperator method contributeRuntimeOperator.

@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
    AbstractUnnestMapOperator unnestMapOp = (AbstractUnnestMapOperator) op;
    ILogicalExpression unnestExpr = unnestMapOp.getExpressionRef().getValue();
    if (unnestExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
        throw new IllegalStateException();
    }
    AbstractFunctionCallExpression unnestFuncExpr = (AbstractFunctionCallExpression) unnestExpr;
    if (unnestFuncExpr.getFunctionIdentifier() != BuiltinFunctions.INDEX_SEARCH) {
        return;
    }
    InvertedIndexJobGenParams jobGenParams = new InvertedIndexJobGenParams();
    jobGenParams.readFromFuncArgs(unnestFuncExpr.getArguments());
    MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
    Dataset dataset;
    try {
        dataset = metadataProvider.findDataset(jobGenParams.getDataverseName(), jobGenParams.getDatasetName());
    } catch (MetadataException e) {
        throw new AlgebricksException(e);
    }
    int[] keyIndexes = getKeyIndexes(jobGenParams.getKeyVarList(), inputSchemas);
    int[] minFilterFieldIndexes = getKeyIndexes(unnestMapOp.getMinFilterVars(), inputSchemas);
    int[] maxFilterFieldIndexes = getKeyIndexes(unnestMapOp.getMaxFilterVars(), inputSchemas);
    boolean retainNull = false;
    if (op.getOperatorTag() == LogicalOperatorTag.LEFT_OUTER_UNNEST_MAP) {
        // By nature, LEFT_OUTER_UNNEST_MAP should generate null values for non-matching tuples.
        retainNull = true;
    }
    // Build runtime.
    Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> invIndexSearch = buildInvertedIndexRuntime(metadataProvider, context, builder.getJobSpec(), unnestMapOp, opSchema, jobGenParams.getRetainInput(), retainNull, jobGenParams.getDatasetName(), dataset, jobGenParams.getIndexName(), jobGenParams.getSearchKeyType(), keyIndexes, jobGenParams.getSearchModifierType(), jobGenParams.getSimilarityThreshold(), minFilterFieldIndexes, maxFilterFieldIndexes, jobGenParams.getIsFullTextSearch());
    // Contribute operator in hyracks job.
    builder.contributeHyracksOperator(unnestMapOp, invIndexSearch.first);
    builder.contributeAlgebricksPartitionConstraint(invIndexSearch.first, invIndexSearch.second);
    ILogicalOperator srcExchange = unnestMapOp.getInputs().get(0).getValue();
    builder.contributeGraphEdge(srcExchange, 0, unnestMapOp, 0);
}
Also used : Dataset(org.apache.asterix.metadata.entities.Dataset) AbstractFunctionCallExpression(org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator) AlgebricksException(org.apache.hyracks.algebricks.common.exceptions.AlgebricksException) MetadataException(org.apache.asterix.metadata.MetadataException) InvertedIndexJobGenParams(org.apache.asterix.optimizer.rules.am.InvertedIndexJobGenParams) ILogicalExpression(org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression) MetadataProvider(org.apache.asterix.metadata.declared.MetadataProvider) IOperatorDescriptor(org.apache.hyracks.api.dataflow.IOperatorDescriptor) AbstractUnnestMapOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator) AlgebricksPartitionConstraint(org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint)

Aggregations

AlgebricksPartitionConstraint (org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint)58 IFileSplitProvider (org.apache.hyracks.dataflow.std.file.IFileSplitProvider)30 IOperatorDescriptor (org.apache.hyracks.api.dataflow.IOperatorDescriptor)23 JobSpecification (org.apache.hyracks.api.job.JobSpecification)23 IIndexDataflowHelperFactory (org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory)22 IndexDataflowHelperFactory (org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory)22 RecordDescriptor (org.apache.hyracks.api.dataflow.value.RecordDescriptor)20 AlgebricksException (org.apache.hyracks.algebricks.common.exceptions.AlgebricksException)19 Pair (org.apache.hyracks.algebricks.common.utils.Pair)19 Index (org.apache.asterix.metadata.entities.Index)15 MetadataException (org.apache.asterix.metadata.MetadataException)14 AlgebricksAbsolutePartitionConstraint (org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint)14 ILogicalOperator (org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator)14 Dataset (org.apache.asterix.metadata.entities.Dataset)12 LogicalVariable (org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable)11 IVariableTypeEnvironment (org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment)10 DatasetCardinalityHint (org.apache.asterix.metadata.dataset.hints.DatasetHints.DatasetCardinalityHint)9 AsterixException (org.apache.asterix.common.exceptions.AsterixException)8 IDataSourceIndex (org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex)8 IMetadataProvider (org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider)8