Examples with PipelinedSorter org.apache.tez.runtime.library.common.sort.impl.PipelinedSorter used on opensource projects

Example 1 with PipelinedSorter

use of org.apache.tez.runtime.library.common.sort.impl.PipelinedSorter in project tez by apache.

the class OrderedPartitionedKVOutput method start.

@Override
public synchronized void start() throws Exception {
    if (!isStarted.get()) {
        memoryUpdateCallbackHandler.validateUpdateReceived();
        String sorterClass = conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS, TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS_DEFAULT).toUpperCase(Locale.ENGLISH);
        SorterImpl sorterImpl = null;
        try {
            sorterImpl = SorterImpl.valueOf(sorterClass);
        } catch (IllegalArgumentException e) {
            throw new IllegalArgumentException("Invalid sorter class specified in config" + ", propertyName=" + TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS + ", value=" + sorterClass + ", validValues=" + Arrays.asList(SorterImpl.values()));
        }
        finalMergeEnabled = conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT_DEFAULT);
        pipelinedShuffle = this.conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED, TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED_DEFAULT);
        if (pipelinedShuffle) {
            if (finalMergeEnabled) {
                LOG.info(getContext().getDestinationVertexName() + " disabling final merge as " + TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED + " is enabled.");
                finalMergeEnabled = false;
                conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, false);
            }
            Preconditions.checkArgument(sorterImpl.equals(SorterImpl.PIPELINED), TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED + "only works with PipelinedSorter.");
        }
        if (sorterImpl.equals(SorterImpl.PIPELINED)) {
            sorter = new PipelinedSorter(getContext(), conf, getNumPhysicalOutputs(), memoryUpdateCallbackHandler.getMemoryAssigned());
        } else if (sorterImpl.equals(SorterImpl.LEGACY)) {
            sorter = new DefaultSorter(getContext(), conf, getNumPhysicalOutputs(), memoryUpdateCallbackHandler.getMemoryAssigned());
        } else {
            throw new UnsupportedOperationException("Unsupported sorter class specified in config" + ", propertyName=" + TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS + ", value=" + sorterClass + ", validValues=" + Arrays.asList(SorterImpl.values()));
        }
        isStarted.set(true);
    }
}