Examples with IAggregatorDescriptor org.apache.hyracks.dataflow.std.group.IAggregatorDescriptor used on opensource projects

Example 1 with IAggregatorDescriptor

use of org.apache.hyracks.dataflow.std.group.IAggregatorDescriptor in project asterixdb by apache.

the class NestedPlansAccumulatingAggregatorFactory method createAggregator.

@Override
public IAggregatorDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDesc, RecordDescriptor outRecordDescriptor, int[] keys, int[] partialKeys) throws HyracksDataException {
    final AggregatorOutput outputWriter = new AggregatorOutput(subplans, keyFieldIdx.length, decorFieldIdx.length);
    final NestedTupleSourceRuntime[] pipelines = new NestedTupleSourceRuntime[subplans.length];
    for (int i = 0; i < subplans.length; i++) {
        pipelines[i] = (NestedTupleSourceRuntime) assemblePipeline(subplans[i], outputWriter, ctx);
    }
    return new IAggregatorDescriptor() {

        @Override
        public void init(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex, AggregateState state) throws HyracksDataException {
            ArrayTupleBuilder tb = outputWriter.getTupleBuilder();
            tb.reset();
            for (int i = 0; i < keyFieldIdx.length; ++i) {
                tb.addField(accessor, tIndex, keyFieldIdx[i]);
            }
            for (int i = 0; i < decorFieldIdx.length; ++i) {
                tb.addField(accessor, tIndex, decorFieldIdx[i]);
            }
            for (int i = 0; i < pipelines.length; ++i) {
                pipelines[i].open();
            }
            // aggregate the first tuple
            for (int i = 0; i < pipelines.length; i++) {
                pipelines[i].writeTuple(accessor.getBuffer(), tIndex);
            }
        }

        @Override
        public void aggregate(IFrameTupleAccessor accessor, int tIndex, IFrameTupleAccessor stateAccessor, int stateTupleIndex, AggregateState state) throws HyracksDataException {
            for (int i = 0; i < pipelines.length; i++) {
                pipelines[i].writeTuple(accessor.getBuffer(), tIndex);
            }
        }

        @Override
        public boolean outputFinalResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor stateAccessor, int tIndex, AggregateState state) throws HyracksDataException {
            for (int i = 0; i < pipelines.length; i++) {
                outputWriter.setInputIdx(i);
                pipelines[i].close();
            }
            // outputWriter.writeTuple(appender);
            tupleBuilder.reset();
            ArrayTupleBuilder tb = outputWriter.getTupleBuilder();
            byte[] data = tb.getByteArray();
            int[] fieldEnds = tb.getFieldEndOffsets();
            int start = 0;
            int offset;
            for (int i = 0; i < fieldEnds.length; i++) {
                if (i > 0) {
                    start = fieldEnds[i - 1];
                }
                offset = fieldEnds[i] - start;
                tupleBuilder.addField(data, start, offset);
            }
            return true;
        }

        @Override
        public AggregateState createAggregateStates() {
            return new AggregateState();
        }

        @Override
        public void reset() {
        }

        @Override
        public boolean outputPartialResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex, AggregateState state) throws HyracksDataException {
            throw new IllegalStateException("this method should not be called");
        }

        @Override
        public void close() {
        }
    };
}

Example 2 with IAggregatorDescriptor

use of org.apache.hyracks.dataflow.std.group.IAggregatorDescriptor in project asterixdb by apache.

the class NestedPlansRunningAggregatorFactory method createAggregator.

/* (non-Javadoc)
     * @see org.apache.hyracks.dataflow.std.group.IAggregatorDescriptorFactory#createAggregator(org.apache.hyracks.api.context.IHyracksTaskContext, org.apache.hyracks.api.dataflow.value.RecordDescriptor, org.apache.hyracks.api.dataflow.value.RecordDescriptor, int[], int[])
     */
@Override
public IAggregatorDescriptor createAggregator(final IHyracksTaskContext ctx, RecordDescriptor inRecordDescriptor, RecordDescriptor outRecordDescriptor, int[] keyFields, int[] keyFieldsInPartialResults, final IFrameWriter writer) throws HyracksDataException {
    final RunningAggregatorOutput outputWriter = new RunningAggregatorOutput(ctx, subplans, keyFieldIdx.length, decorFieldIdx.length, writer);
    final NestedTupleSourceRuntime[] pipelines = new NestedTupleSourceRuntime[subplans.length];
    for (int i = 0; i < subplans.length; i++) {
        pipelines[i] = (NestedTupleSourceRuntime) assemblePipeline(subplans[i], outputWriter, ctx);
    }
    final ArrayTupleBuilder gbyTb = outputWriter.getGroupByTupleBuilder();
    return new IAggregatorDescriptor() {

        @Override
        public void init(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex, AggregateState state) throws HyracksDataException {
            for (int i = 0; i < pipelines.length; ++i) {
                pipelines[i].open();
            }
            gbyTb.reset();
            for (int i = 0; i < keyFieldIdx.length; ++i) {
                gbyTb.addField(accessor, tIndex, keyFieldIdx[i]);
            }
            for (int i = 0; i < decorFieldIdx.length; ++i) {
                gbyTb.addField(accessor, tIndex, decorFieldIdx[i]);
            }
            // aggregate the first tuple
            for (int i = 0; i < pipelines.length; i++) {
                outputWriter.setInputIdx(i);
                pipelines[i].writeTuple(accessor.getBuffer(), tIndex);
            }
        }

        @Override
        public void aggregate(IFrameTupleAccessor accessor, int tIndex, IFrameTupleAccessor stateAccessor, int stateTupleIndex, AggregateState state) throws HyracksDataException {
            for (int i = 0; i < pipelines.length; i++) {
                outputWriter.setInputIdx(i);
                pipelines[i].writeTuple(accessor.getBuffer(), tIndex);
            }
        }

        @Override
        public boolean outputFinalResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex, AggregateState state) throws HyracksDataException {
            for (int i = 0; i < pipelines.length; ++i) {
                outputWriter.setInputIdx(i);
                pipelines[i].close();
            }
            return false;
        }

        @Override
        public AggregateState createAggregateStates() {
            return new AggregateState();
        }

        @Override
        public void reset() {
        }

        @Override
        public boolean outputPartialResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex, AggregateState state) throws HyracksDataException {
            throw new IllegalStateException("this method should not be called");
        }

        @Override
        public void close() {
        }
    };
}

Example 3 with IAggregatorDescriptor

use of org.apache.hyracks.dataflow.std.group.IAggregatorDescriptor in project asterixdb by apache.

the class SerializableAggregatorDescriptorFactory method createAggregator.

@Override
public IAggregatorDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDescriptor, RecordDescriptor outRecordDescriptor, int[] keyFields, final int[] keyFieldsInPartialResults) throws HyracksDataException {
    final int[] keys = keyFields;
    /**
         * one IAggregatorDescriptor instance per Gby operator
         */
    return new IAggregatorDescriptor() {

        private FrameTupleReference ftr = new FrameTupleReference();

        private ISerializedAggregateEvaluator[] aggs = new ISerializedAggregateEvaluator[aggFactories.length];

        private int offsetFieldIndex = keys.length;

        private int[] stateFieldLength = new int[aggFactories.length];

        @Override
        public AggregateState createAggregateStates() {
            return new AggregateState();
        }

        @Override
        public void init(ArrayTupleBuilder tb, IFrameTupleAccessor accessor, int tIndex, AggregateState state) throws HyracksDataException {
            DataOutput output = tb.getDataOutput();
            ftr.reset(accessor, tIndex);
            for (int i = 0; i < aggs.length; i++) {
                int begin = tb.getSize();
                if (aggs[i] == null) {
                    aggs[i] = aggFactories[i].createAggregateEvaluator(ctx);
                }
                aggs[i].init(output);
                tb.addFieldEndOffset();
                stateFieldLength[i] = tb.getSize() - begin;
            }
            // doing initial aggregate
            ftr.reset(accessor, tIndex);
            for (int i = 0; i < aggs.length; i++) {
                byte[] data = tb.getByteArray();
                int prevFieldPos = i + keys.length - 1;
                int start = prevFieldPos >= 0 ? tb.getFieldEndOffsets()[prevFieldPos] : 0;
                aggs[i].step(ftr, data, start, stateFieldLength[i]);
            }
        }

        @Override
        public void aggregate(IFrameTupleAccessor accessor, int tIndex, IFrameTupleAccessor stateAccessor, int stateTupleIndex, AggregateState state) throws HyracksDataException {
            ftr.reset(accessor, tIndex);
            int stateTupleStart = stateAccessor.getTupleStartOffset(stateTupleIndex);
            int fieldSlotLength = stateAccessor.getFieldSlotsLength();
            for (int i = 0; i < aggs.length; i++) {
                byte[] data = stateAccessor.getBuffer().array();
                int start = stateAccessor.getFieldStartOffset(stateTupleIndex, i + keys.length) + stateTupleStart + fieldSlotLength;
                aggs[i].step(ftr, data, start, stateFieldLength[i]);
            }
        }

        @Override
        public boolean outputPartialResult(ArrayTupleBuilder tb, IFrameTupleAccessor stateAccessor, int tIndex, AggregateState state) throws HyracksDataException {
            byte[] data = stateAccessor.getBuffer().array();
            int startOffset = stateAccessor.getTupleStartOffset(tIndex);
            int aggFieldOffset = stateAccessor.getFieldStartOffset(tIndex, offsetFieldIndex);
            int refOffset = startOffset + stateAccessor.getFieldSlotsLength() + aggFieldOffset;
            int start = refOffset;
            for (int i = 0; i < aggs.length; i++) {
                aggs[i].finishPartial(data, start, stateFieldLength[i], tb.getDataOutput());
                start += stateFieldLength[i];
                tb.addFieldEndOffset();
            }
            return true;
        }

        @Override
        public boolean outputFinalResult(ArrayTupleBuilder tb, IFrameTupleAccessor stateAccessor, int tIndex, AggregateState state) throws HyracksDataException {
            byte[] data = stateAccessor.getBuffer().array();
            int startOffset = stateAccessor.getTupleStartOffset(tIndex);
            int aggFieldOffset = stateAccessor.getFieldStartOffset(tIndex, offsetFieldIndex);
            int refOffset = startOffset + stateAccessor.getFieldSlotsLength() + aggFieldOffset;
            int start = refOffset;
            for (int i = 0; i < aggs.length; i++) {
                aggs[i].finish(data, start, stateFieldLength[i], tb.getDataOutput());
                start += stateFieldLength[i];
                tb.addFieldEndOffset();
            }
            return true;
        }

        @Override
        public void reset() {
        }

        @Override
        public void close() {
            reset();
        }
    };
}

Example 4 with IAggregatorDescriptor

use of org.apache.hyracks.dataflow.std.group.IAggregatorDescriptor in project asterixdb by apache.

the class MultiFieldsAggregatorFactory method createAggregator.

/*
     * (non-Javadoc)
     *
     * @see
     * org.apache.hyracks.dataflow.std.aggregations.IAggregatorDescriptorFactory
     * #createAggregator(org.apache.hyracks.api.context.IHyracksTaskContext,
     * org.apache.hyracks.api.dataflow.value.RecordDescriptor,
     * org.apache.hyracks.api.dataflow.value.RecordDescriptor)
     */
@Override
public IAggregatorDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDescriptor, RecordDescriptor outRecordDescriptor, final int[] keyFields, final int[] keyFieldsInPartialResults) throws HyracksDataException {
    final IFieldAggregateDescriptor[] aggregators = new IFieldAggregateDescriptor[aggregatorFactories.length];
    for (int i = 0; i < aggregators.length; i++) {
        aggregators[i] = aggregatorFactories[i].createAggregator(ctx, inRecordDescriptor, outRecordDescriptor);
    }
    if (this.keys == null) {
        this.keys = keyFields;
    }
    return new IAggregatorDescriptor() {

        @Override
        public void reset() {
            for (int i = 0; i < aggregators.length; i++) {
                aggregators[i].reset();
            }
        }

        @Override
        public boolean outputPartialResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor stateAccessor, int tIndex, AggregateState state) throws HyracksDataException {
            DataOutput dos = tupleBuilder.getDataOutput();
            int tupleOffset = stateAccessor.getTupleStartOffset(tIndex);
            for (int i = 0; i < aggregators.length; i++) {
                int fieldOffset = stateAccessor.getFieldStartOffset(tIndex, keys.length + i);
                aggregators[i].outputPartialResult(dos, stateAccessor.getBuffer().array(), fieldOffset + stateAccessor.getFieldSlotsLength() + tupleOffset, ((AggregateState[]) state.state)[i]);
                tupleBuilder.addFieldEndOffset();
            }
            return true;
        }

        @Override
        public boolean outputFinalResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor stateAccessor, int tIndex, AggregateState state) throws HyracksDataException {
            DataOutput dos = tupleBuilder.getDataOutput();
            int tupleOffset = stateAccessor.getTupleStartOffset(tIndex);
            for (int i = 0; i < aggregators.length; i++) {
                if (aggregators[i].needsBinaryState()) {
                    int fieldOffset = stateAccessor.getFieldStartOffset(tIndex, keys.length + i);
                    aggregators[i].outputFinalResult(dos, stateAccessor.getBuffer().array(), tupleOffset + stateAccessor.getFieldSlotsLength() + fieldOffset, ((AggregateState[]) state.state)[i]);
                } else {
                    aggregators[i].outputFinalResult(dos, null, 0, ((AggregateState[]) state.state)[i]);
                }
                tupleBuilder.addFieldEndOffset();
            }
            return true;
        }

        @Override
        public void init(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex, AggregateState state) throws HyracksDataException {
            DataOutput dos = tupleBuilder.getDataOutput();
            for (int i = 0; i < aggregators.length; i++) {
                aggregators[i].init(accessor, tIndex, dos, ((AggregateState[]) state.state)[i]);
                if (aggregators[i].needsBinaryState()) {
                    tupleBuilder.addFieldEndOffset();
                }
            }
        }

        @Override
        public AggregateState createAggregateStates() {
            AggregateState[] states = new AggregateState[aggregators.length];
            for (int i = 0; i < states.length; i++) {
                states[i] = aggregators[i].createState();
            }
            return new AggregateState(states);
        }

        @Override
        public void close() {
            for (int i = 0; i < aggregators.length; i++) {
                aggregators[i].close();
            }
        }

        @Override
        public void aggregate(IFrameTupleAccessor accessor, int tIndex, IFrameTupleAccessor stateAccessor, int stateTupleIndex, AggregateState state) throws HyracksDataException {
            if (stateAccessor != null) {
                int stateTupleOffset = stateAccessor.getTupleStartOffset(stateTupleIndex);
                int fieldIndex = 0;
                for (int i = 0; i < aggregators.length; i++) {
                    if (aggregators[i].needsBinaryState()) {
                        int stateFieldOffset = stateAccessor.getFieldStartOffset(stateTupleIndex, keys.length + fieldIndex);
                        aggregators[i].aggregate(accessor, tIndex, stateAccessor.getBuffer().array(), stateTupleOffset + stateAccessor.getFieldSlotsLength() + stateFieldOffset, ((AggregateState[]) state.state)[i]);
                        fieldIndex++;
                    } else {
                        aggregators[i].aggregate(accessor, tIndex, null, 0, ((AggregateState[]) state.state)[i]);
                    }
                }
            } else {
                for (int i = 0; i < aggregators.length; i++) {
                    aggregators[i].aggregate(accessor, tIndex, null, 0, ((AggregateState[]) state.state)[i]);
                }
            }
        }
    };
}

Example 5 with IAggregatorDescriptor

use of org.apache.hyracks.dataflow.std.group.IAggregatorDescriptor in project asterixdb by apache.

the class SimpleAlgebricksAccumulatingAggregatorFactory method createAggregator.

@Override
public IAggregatorDescriptor createAggregator(final IHyracksTaskContext ctx, RecordDescriptor inRecordDesc, RecordDescriptor outRecordDescriptor, int[] aggKeys, int[] partialKeys) throws HyracksDataException {
    return new IAggregatorDescriptor() {

        private FrameTupleReference ftr = new FrameTupleReference();

        private IPointable p = VoidPointable.FACTORY.createPointable();

        @Override
        public void init(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex, AggregateState state) throws HyracksDataException {
            IAggregateEvaluator[] agg = (IAggregateEvaluator[]) state.state;
            // initialize aggregate functions
            for (int i = 0; i < agg.length; i++) {
                agg[i].init();
            }
            ftr.reset(accessor, tIndex);
            for (int i = 0; i < agg.length; i++) {
                agg[i].step(ftr);
            }
        }

        @Override
        public void aggregate(IFrameTupleAccessor accessor, int tIndex, IFrameTupleAccessor stateAccessor, int stateTupleIndex, AggregateState state) throws HyracksDataException {
            IAggregateEvaluator[] agg = (IAggregateEvaluator[]) state.state;
            ftr.reset(accessor, tIndex);
            for (int i = 0; i < agg.length; i++) {
                agg[i].step(ftr);
            }
        }

        @Override
        public boolean outputFinalResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor stateAccessor, int tIndex, AggregateState state) throws HyracksDataException {
            IAggregateEvaluator[] agg = (IAggregateEvaluator[]) state.state;
            for (int i = 0; i < agg.length; i++) {
                agg[i].finish(p);
                tupleBuilder.addField(p.getByteArray(), p.getStartOffset(), p.getLength());
            }
            return true;
        }

        @Override
        public AggregateState createAggregateStates() throws HyracksDataException {
            IAggregateEvaluator[] agg = new IAggregateEvaluator[aggFactories.length];
            for (int i = 0; i < agg.length; i++) {
                agg[i] = aggFactories[i].createAggregateEvaluator(ctx);
            }
            return new AggregateState(agg);
        }

        @Override
        public void reset() {
        }

        @Override
        public boolean outputPartialResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor stateAccessor, int tIndex, AggregateState state) throws HyracksDataException {
            IAggregateEvaluator[] agg = (IAggregateEvaluator[]) state.state;
            for (int i = 0; i < agg.length; i++) {
                agg[i].finishPartial(p);
                tupleBuilder.addField(p.getByteArray(), p.getStartOffset(), p.getLength());
            }
            return true;
        }

        @Override
        public void close() {
        }
    };
}