Search in sources :

Example 1 with INormalizedKeyComputer

use of org.apache.hyracks.api.dataflow.value.INormalizedKeyComputer in project asterixdb by apache.

the class SortGroupbyTest method initial.

@Override
protected void initial(final IHyracksTaskContext ctx, int tableSize, final int numFrames) throws HyracksDataException {
    builder = new ExternalSortGroupByRunGenerator(ctx, keyFields, inRecordDesc, numFrames, keyFields, normalizedKeyComputerFactory, comparatorFactories, partialAggrInState, outputRec, Algorithm.QUICK_SORT);
    mergerOperator = new AbstractUnaryOutputSourceOperatorNodePushable() {

        @Override
        public void initialize() throws HyracksDataException {
            List<GeneratedRunFileReader> runs = builder.getRuns();
            ISorter sorter = builder.getSorter();
            IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
            for (int i = 0; i < comparatorFactories.length; ++i) {
                comparators[i] = comparatorFactories[i].createBinaryComparator();
            }
            INormalizedKeyComputer nmkComputer = normalizedKeyComputerFactory == null ? null : normalizedKeyComputerFactory.createNormalizedKeyComputer();
            AbstractExternalSortRunMerger merger = new ExternalSortGroupByRunMerger(ctx, sorter, runs, keyFields, inRecordDesc, outputRec, outputRec, numFrames, writer, keyFields, nmkComputer, comparators, partialAggrInState, finalAggrInState, true);
            merger.process();
        }
    };
}
Also used : INormalizedKeyComputer(org.apache.hyracks.api.dataflow.value.INormalizedKeyComputer) AbstractUnaryOutputSourceOperatorNodePushable(org.apache.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable) AbstractExternalSortRunMerger(org.apache.hyracks.dataflow.std.sort.AbstractExternalSortRunMerger) List(java.util.List) ExternalSortGroupByRunMerger(org.apache.hyracks.dataflow.std.group.sort.ExternalSortGroupByRunMerger) ExternalSortGroupByRunGenerator(org.apache.hyracks.dataflow.std.group.sort.ExternalSortGroupByRunGenerator) HyracksDataException(org.apache.hyracks.api.exceptions.HyracksDataException) ISorter(org.apache.hyracks.dataflow.std.sort.ISorter)

Example 2 with INormalizedKeyComputer

use of org.apache.hyracks.api.dataflow.value.INormalizedKeyComputer in project asterixdb by apache.

the class MToNPartitioningMergingConnectorDescriptor method createPartitionCollector.

@Override
public IPartitionCollector createPartitionCollector(IHyracksTaskContext ctx, RecordDescriptor recordDesc, int index, int nProducerPartitions, int nConsumerPartitions) throws HyracksDataException {
    IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
    for (int i = 0; i < comparatorFactories.length; ++i) {
        comparators[i] = comparatorFactories[i].createBinaryComparator();
    }
    INormalizedKeyComputer nmkComputer = nkcFactory == null ? null : nkcFactory.createNormalizedKeyComputer();
    IPartitionBatchManager pbm = new NonDeterministicPartitionBatchManager(nProducerPartitions);
    IFrameReader sortMergeFrameReader = new SortMergeFrameReader(ctx, nProducerPartitions, nProducerPartitions, sortFields, comparators, nmkComputer, recordDesc, pbm);
    BitSet expectedPartitions = new BitSet();
    expectedPartitions.set(0, nProducerPartitions);
    return new PartitionCollector(ctx, getConnectorId(), index, expectedPartitions, sortMergeFrameReader, pbm);
}
Also used : INormalizedKeyComputer(org.apache.hyracks.api.dataflow.value.INormalizedKeyComputer) SortMergeFrameReader(org.apache.hyracks.dataflow.std.collectors.SortMergeFrameReader) IFrameReader(org.apache.hyracks.api.comm.IFrameReader) IPartitionCollector(org.apache.hyracks.api.comm.IPartitionCollector) PartitionCollector(org.apache.hyracks.dataflow.std.collectors.PartitionCollector) IPartitionBatchManager(org.apache.hyracks.dataflow.std.collectors.IPartitionBatchManager) BitSet(java.util.BitSet) IBinaryComparator(org.apache.hyracks.api.dataflow.value.IBinaryComparator) NonDeterministicPartitionBatchManager(org.apache.hyracks.dataflow.std.collectors.NonDeterministicPartitionBatchManager)

Aggregations

INormalizedKeyComputer (org.apache.hyracks.api.dataflow.value.INormalizedKeyComputer)2 BitSet (java.util.BitSet)1 List (java.util.List)1 IFrameReader (org.apache.hyracks.api.comm.IFrameReader)1 IPartitionCollector (org.apache.hyracks.api.comm.IPartitionCollector)1 IBinaryComparator (org.apache.hyracks.api.dataflow.value.IBinaryComparator)1 HyracksDataException (org.apache.hyracks.api.exceptions.HyracksDataException)1 AbstractUnaryOutputSourceOperatorNodePushable (org.apache.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable)1 IPartitionBatchManager (org.apache.hyracks.dataflow.std.collectors.IPartitionBatchManager)1 NonDeterministicPartitionBatchManager (org.apache.hyracks.dataflow.std.collectors.NonDeterministicPartitionBatchManager)1 PartitionCollector (org.apache.hyracks.dataflow.std.collectors.PartitionCollector)1 SortMergeFrameReader (org.apache.hyracks.dataflow.std.collectors.SortMergeFrameReader)1 ExternalSortGroupByRunGenerator (org.apache.hyracks.dataflow.std.group.sort.ExternalSortGroupByRunGenerator)1 ExternalSortGroupByRunMerger (org.apache.hyracks.dataflow.std.group.sort.ExternalSortGroupByRunMerger)1 AbstractExternalSortRunMerger (org.apache.hyracks.dataflow.std.sort.AbstractExternalSortRunMerger)1 ISorter (org.apache.hyracks.dataflow.std.sort.ISorter)1