Examples with DefaultSorter org.apache.tez.runtime.library.common.sort.impl.dflt.DefaultSorter used on opensource projects

Example 1 with DefaultSorter

use of org.apache.tez.runtime.library.common.sort.impl.dflt.DefaultSorter in project tez by apache.

the class TestOnFileSortedOutput method testSortBufferSize.

@Test(timeout = 5000)
public void testSortBufferSize() throws Exception {
    OutputContext context = createTezOutputContext();
    conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 2048);
    UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf);
    doReturn(payLoad).when(context).getUserPayload();
    sortedOutput = new OrderedPartitionedKVOutput(context, partitions);
    try {
        // Memory limit checks are done in sorter impls. For e.g, defaultsorter does not support > 2GB
        sortedOutput.initialize();
        DefaultSorter sorter = new DefaultSorter(context, conf, 100, 3500 * 1024 * 1024l);
        fail();
    } catch (IllegalArgumentException e) {
        assertTrue(e.getMessage().contains(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB));
    }
    conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 0);
    payLoad = TezUtils.createUserPayloadFromConf(conf);
    doReturn(payLoad).when(context).getUserPayload();
    sortedOutput = new OrderedPartitionedKVOutput(context, partitions);
    try {
        sortedOutput.initialize();
        fail();
    } catch (IllegalArgumentException e) {
        assertTrue(e.getMessage().contains(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB));
    }
}

Example 2 with DefaultSorter

use of org.apache.tez.runtime.library.common.sort.impl.dflt.DefaultSorter in project tez by apache.

the class OrderedPartitionedKVOutput method start.

@Override
public synchronized void start() throws Exception {
    if (!isStarted.get()) {
        memoryUpdateCallbackHandler.validateUpdateReceived();
        String sorterClass = conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS, TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS_DEFAULT).toUpperCase(Locale.ENGLISH);
        SorterImpl sorterImpl = null;
        try {
            sorterImpl = SorterImpl.valueOf(sorterClass);
        } catch (IllegalArgumentException e) {
            throw new IllegalArgumentException("Invalid sorter class specified in config" + ", propertyName=" + TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS + ", value=" + sorterClass + ", validValues=" + Arrays.asList(SorterImpl.values()));
        }
        finalMergeEnabled = conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT_DEFAULT);
        pipelinedShuffle = this.conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED, TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED_DEFAULT);
        if (pipelinedShuffle) {
            if (finalMergeEnabled) {
                LOG.info(getContext().getDestinationVertexName() + " disabling final merge as " + TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED + " is enabled.");
                finalMergeEnabled = false;
                conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, false);
            }
            Preconditions.checkArgument(sorterImpl.equals(SorterImpl.PIPELINED), TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED + "only works with PipelinedSorter.");
        }
        if (sorterImpl.equals(SorterImpl.PIPELINED)) {
            sorter = new PipelinedSorter(getContext(), conf, getNumPhysicalOutputs(), memoryUpdateCallbackHandler.getMemoryAssigned());
        } else if (sorterImpl.equals(SorterImpl.LEGACY)) {
            sorter = new DefaultSorter(getContext(), conf, getNumPhysicalOutputs(), memoryUpdateCallbackHandler.getMemoryAssigned());
        } else {
            throw new UnsupportedOperationException("Unsupported sorter class specified in config" + ", propertyName=" + TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS + ", value=" + sorterClass + ", validValues=" + Arrays.asList(SorterImpl.values()));
        }
        isStarted.set(true);
    }
}