Examples with CombiningUnilateralSortMerger org.apache.flink.runtime.operators.sort.CombiningUnilateralSortMerger used on opensource projects

Example 1 with CombiningUnilateralSortMerger

use of org.apache.flink.runtime.operators.sort.CombiningUnilateralSortMerger in project flink by apache.

the class BatchTask method initInputLocalStrategy.

private void initInputLocalStrategy(int inputNum) throws Exception {
    // check if there is already a strategy
    if (this.localStrategies[inputNum] != null) {
        throw new IllegalStateException();
    }
    // now set up the local strategy
    final LocalStrategy localStrategy = this.config.getInputLocalStrategy(inputNum);
    if (localStrategy != null) {
        switch(localStrategy) {
            case NONE:
                // the input is as it is
                this.inputs[inputNum] = this.inputIterators[inputNum];
                break;
            case SORT:
                @SuppressWarnings({ "rawtypes", "unchecked" }) UnilateralSortMerger<?> sorter = new UnilateralSortMerger(getMemoryManager(), getIOManager(), this.inputIterators[inputNum], this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum), this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum), this.config.getSpillingThresholdInput(inputNum), this.config.getUseLargeRecordHandler(), this.getExecutionConfig().isObjectReuseEnabled());
                // set the input to null such that it will be lazily fetched from the input strategy
                this.inputs[inputNum] = null;
                this.localStrategies[inputNum] = sorter;
                break;
            case COMBININGSORT:
                // we should have nested configurations for the local strategies to solve that
                if (inputNum != 0) {
                    throw new IllegalStateException("Performing combining sort outside a (group)reduce task!");
                }
                // instantiate ourselves a combiner. we should not use the stub, because the sort and the
                // subsequent (group)reduce would otherwise share it multi-threaded
                final Class<S> userCodeFunctionType = this.driver.getStubType();
                if (userCodeFunctionType == null) {
                    throw new IllegalStateException("Performing combining sort outside a reduce task!");
                }
                final S localStub;
                try {
                    localStub = initStub(userCodeFunctionType);
                } catch (Exception e) {
                    throw new RuntimeException("Initializing the user code and the configuration failed" + (e.getMessage() == null ? "." : ": " + e.getMessage()), e);
                }
                if (!(localStub instanceof GroupCombineFunction)) {
                    throw new IllegalStateException("Performing combining sort outside a reduce task!");
                }
                @SuppressWarnings({ "rawtypes", "unchecked" }) CombiningUnilateralSortMerger<?> cSorter = new CombiningUnilateralSortMerger((GroupCombineFunction) localStub, getMemoryManager(), getIOManager(), this.inputIterators[inputNum], this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum), this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum), this.config.getSpillingThresholdInput(inputNum), this.getTaskConfig().getUseLargeRecordHandler(), this.getExecutionConfig().isObjectReuseEnabled());
                cSorter.setUdfConfiguration(this.config.getStubParameters());
                // set the input to null such that it will be lazily fetched from the input strategy
                this.inputs[inputNum] = null;
                this.localStrategies[inputNum] = cSorter;
                break;
            default:
                throw new Exception("Unrecognized local strategy provided: " + localStrategy.name());
        }
    } else {
        // no local strategy in the config
        this.inputs[inputNum] = this.inputIterators[inputNum];
    }
}