Search in sources :

Example 6 with IOperatorDescriptorRegistry

use of org.apache.hyracks.api.job.IOperatorDescriptorRegistry in project asterixdb by apache.

the class StableSortPOperator method contributeRuntimeOperator.

@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
    IOperatorDescriptorRegistry spec = builder.getJobSpec();
    RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
    int n = sortColumns.length;
    int[] sortFields = new int[n];
    IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
    INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
    INormalizedKeyComputerFactory nkcf = null;
    IVariableTypeEnvironment env = context.getTypeEnvironment(op);
    int i = 0;
    for (OrderColumn oc : sortColumns) {
        LogicalVariable var = oc.getColumn();
        sortFields[i] = opSchema.findVariable(var);
        Object type = env.getVarType(var);
        OrderKind order = oc.getOrder();
        if (i == 0 && nkcfProvider != null && type != null) {
            nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, order == OrderKind.ASC);
        }
        IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
        comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder() == OrderKind.ASC);
        i++;
    }
    // topK == -1 means that a topK value is not provided.
    if (topK == -1) {
        ExternalSortOperatorDescriptor sortOpDesc = new ExternalSortOperatorDescriptor(spec, maxNumberOfFrames, sortFields, nkcf, comps, recDescriptor);
        contributeOpDesc(builder, (AbstractLogicalOperator) op, sortOpDesc);
        ILogicalOperator src = op.getInputs().get(0).getValue();
        builder.contributeGraphEdge(src, 0, op, 0);
    } else {
        // Since topK value is provided, topK optimization is possible.
        // We call topKSorter instead of calling ExternalSortOperator.
        TopKSorterOperatorDescriptor sortOpDesc = new TopKSorterOperatorDescriptor(spec, maxNumberOfFrames, topK, sortFields, nkcf, comps, recDescriptor);
        contributeOpDesc(builder, (AbstractLogicalOperator) op, sortOpDesc);
        ILogicalOperator src = op.getInputs().get(0).getValue();
        builder.contributeGraphEdge(src, 0, op, 0);
    }
}
Also used : LogicalVariable(org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable) RecordDescriptor(org.apache.hyracks.api.dataflow.value.RecordDescriptor) OrderColumn(org.apache.hyracks.algebricks.core.algebra.properties.OrderColumn) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator) IBinaryComparatorFactory(org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory) IOperatorDescriptorRegistry(org.apache.hyracks.api.job.IOperatorDescriptorRegistry) IBinaryComparatorFactoryProvider(org.apache.hyracks.algebricks.data.IBinaryComparatorFactoryProvider) TopKSorterOperatorDescriptor(org.apache.hyracks.dataflow.std.sort.TopKSorterOperatorDescriptor) INormalizedKeyComputerFactory(org.apache.hyracks.api.dataflow.value.INormalizedKeyComputerFactory) OrderKind(org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind) INormalizedKeyComputerFactoryProvider(org.apache.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider) ExternalSortOperatorDescriptor(org.apache.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor) IVariableTypeEnvironment(org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment)

Example 7 with IOperatorDescriptorRegistry

use of org.apache.hyracks.api.job.IOperatorDescriptorRegistry in project asterixdb by apache.

the class PreSortedDistinctByPOperator method contributeRuntimeOperator.

@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
    IOperatorDescriptorRegistry spec = builder.getJobSpec();
    int[] keys = JobGenHelper.variablesToFieldIndexes(columnList, inputSchemas[0]);
    int sz = inputSchemas[0].getSize();
    int fdSz = sz - columnList.size();
    int[] fdColumns = new int[fdSz];
    int j = 0;
    for (LogicalVariable v : inputSchemas[0]) {
        if (!columnList.contains(v)) {
            fdColumns[j++] = inputSchemas[0].findVariable(v);
        }
    }
    int[] keysAndDecs = new int[keys.length + fdColumns.length];
    for (int i = 0; i < keys.length; i++) {
        keysAndDecs[i] = keys[i];
    }
    for (int i = 0; i < fdColumns.length; i++) {
        keysAndDecs[i + keys.length] = fdColumns[i];
    }
    IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(columnList, context.getTypeEnvironment(op), context);
    IAggregateEvaluatorFactory[] aggFactories = new IAggregateEvaluatorFactory[] {};
    IAggregatorDescriptorFactory aggregatorFactory = new SimpleAlgebricksAccumulatingAggregatorFactory(aggFactories, keysAndDecs);
    RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
    /** make fd columns part of the key but the comparator only compares the distinct key columns */
    PreclusteredGroupOperatorDescriptor opDesc = new PreclusteredGroupOperatorDescriptor(spec, keysAndDecs, comparatorFactories, aggregatorFactory, recordDescriptor);
    contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
    ILogicalOperator src = op.getInputs().get(0).getValue();
    builder.contributeGraphEdge(src, 0, op, 0);
}
Also used : LogicalVariable(org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable) RecordDescriptor(org.apache.hyracks.api.dataflow.value.RecordDescriptor) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator) IBinaryComparatorFactory(org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory) SimpleAlgebricksAccumulatingAggregatorFactory(org.apache.hyracks.algebricks.runtime.operators.aggreg.SimpleAlgebricksAccumulatingAggregatorFactory) IOperatorDescriptorRegistry(org.apache.hyracks.api.job.IOperatorDescriptorRegistry) IAggregateEvaluatorFactory(org.apache.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory) IAggregatorDescriptorFactory(org.apache.hyracks.dataflow.std.group.IAggregatorDescriptorFactory) PreclusteredGroupOperatorDescriptor(org.apache.hyracks.dataflow.std.group.preclustered.PreclusteredGroupOperatorDescriptor)

Example 8 with IOperatorDescriptorRegistry

use of org.apache.hyracks.api.job.IOperatorDescriptorRegistry in project asterixdb by apache.

the class PreclusteredGroupByPOperator method contributeRuntimeOperator.

@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
    int[] keys = JobGenHelper.variablesToFieldIndexes(columnList, inputSchemas[0]);
    GroupByOperator gby = (GroupByOperator) op;
    int[] fdColumns = getFdColumns(gby, inputSchemas[0]);
    // compile subplans and set the gby op. schema accordingly
    AlgebricksPipeline[] subplans = compileSubplans(inputSchemas[0], gby, opSchema, context);
    IAggregatorDescriptorFactory aggregatorFactory;
    if (gby.getNestedPlans().get(0).getRoots().get(0).getValue().getOperatorTag() == LogicalOperatorTag.RUNNINGAGGREGATE) {
        aggregatorFactory = new NestedPlansRunningAggregatorFactory(subplans, keys, fdColumns);
    } else {
        aggregatorFactory = new NestedPlansAccumulatingAggregatorFactory(subplans, keys, fdColumns);
    }
    IOperatorDescriptorRegistry spec = builder.getJobSpec();
    IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(columnList, context.getTypeEnvironment(op), context);
    RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
    PreclusteredGroupOperatorDescriptor opDesc = new PreclusteredGroupOperatorDescriptor(spec, keys, comparatorFactories, aggregatorFactory, recordDescriptor, groupAll);
    contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
    ILogicalOperator src = op.getInputs().get(0).getValue();
    builder.contributeGraphEdge(src, 0, op, 0);
}
Also used : GroupByOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator) RecordDescriptor(org.apache.hyracks.api.dataflow.value.RecordDescriptor) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator) IBinaryComparatorFactory(org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory) NestedPlansAccumulatingAggregatorFactory(org.apache.hyracks.algebricks.runtime.operators.aggreg.NestedPlansAccumulatingAggregatorFactory) IOperatorDescriptorRegistry(org.apache.hyracks.api.job.IOperatorDescriptorRegistry) IAggregatorDescriptorFactory(org.apache.hyracks.dataflow.std.group.IAggregatorDescriptorFactory) NestedPlansRunningAggregatorFactory(org.apache.hyracks.algebricks.runtime.operators.aggreg.NestedPlansRunningAggregatorFactory) PreclusteredGroupOperatorDescriptor(org.apache.hyracks.dataflow.std.group.preclustered.PreclusteredGroupOperatorDescriptor) AlgebricksPipeline(org.apache.hyracks.algebricks.runtime.base.AlgebricksPipeline)

Example 9 with IOperatorDescriptorRegistry

use of org.apache.hyracks.api.job.IOperatorDescriptorRegistry in project asterixdb by apache.

the class SinkPOperator method contributeRuntimeOperator.

@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
    IOperatorDescriptorRegistry spec = builder.getJobSpec();
    SinkOperatorDescriptor opDesc = new SinkOperatorDescriptor(spec, op.getInputs().size());
    contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
    for (int i = 0; i < op.getInputs().size(); i++) {
        builder.contributeGraphEdge(op.getInputs().get(i).getValue(), 0, op, i);
    }
}
Also used : SinkOperatorDescriptor(org.apache.hyracks.dataflow.std.misc.SinkOperatorDescriptor) IOperatorDescriptorRegistry(org.apache.hyracks.api.job.IOperatorDescriptorRegistry)

Example 10 with IOperatorDescriptorRegistry

use of org.apache.hyracks.api.job.IOperatorDescriptorRegistry in project asterixdb by apache.

the class UnionAllPOperator method contributeRuntimeOperator.

@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
    IOperatorDescriptorRegistry spec = builder.getJobSpec();
    RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
    // at algebricks level, union all only accepts two inputs, although at
    // hyracks
    // level, there is no restrictions
    UnionAllOperatorDescriptor opDesc = new UnionAllOperatorDescriptor(spec, 2, recordDescriptor);
    contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
    ILogicalOperator src1 = op.getInputs().get(0).getValue();
    builder.contributeGraphEdge(src1, 0, op, 0);
    ILogicalOperator src2 = op.getInputs().get(1).getValue();
    builder.contributeGraphEdge(src2, 0, op, 1);
}
Also used : RecordDescriptor(org.apache.hyracks.api.dataflow.value.RecordDescriptor) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator) IOperatorDescriptorRegistry(org.apache.hyracks.api.job.IOperatorDescriptorRegistry) UnionAllOperatorDescriptor(org.apache.hyracks.dataflow.std.union.UnionAllOperatorDescriptor)

Aggregations

IOperatorDescriptorRegistry (org.apache.hyracks.api.job.IOperatorDescriptorRegistry)14 RecordDescriptor (org.apache.hyracks.api.dataflow.value.RecordDescriptor)12 ILogicalOperator (org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator)11 IBinaryComparatorFactory (org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory)8 LogicalVariable (org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable)6 IVariableTypeEnvironment (org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment)5 IExpressionRuntimeProvider (org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider)4 IBinaryComparatorFactoryProvider (org.apache.hyracks.algebricks.data.IBinaryComparatorFactoryProvider)4 INormalizedKeyComputerFactory (org.apache.hyracks.api.dataflow.value.INormalizedKeyComputerFactory)4 IAggregatorDescriptorFactory (org.apache.hyracks.dataflow.std.group.IAggregatorDescriptorFactory)4 AlgebricksException (org.apache.hyracks.algebricks.common.exceptions.AlgebricksException)3 GroupByOperator (org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator)3 IOperatorSchema (org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema)3 INormalizedKeyComputerFactoryProvider (org.apache.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider)3 IOperatorDescriptor (org.apache.hyracks.api.dataflow.IOperatorDescriptor)3 ArrayList (java.util.ArrayList)2 Mutable (org.apache.commons.lang3.mutable.Mutable)2 NotImplementedException (org.apache.hyracks.algebricks.common.exceptions.NotImplementedException)2 ILogicalExpression (org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression)2 ILogicalPlan (org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan)2