Examples with ShuffleManager org.apache.tez.runtime.library.common.shuffle.impl.ShuffleManager used on opensource projects

Example 1 with ShuffleManager

use of org.apache.tez.runtime.library.common.shuffle.impl.ShuffleManager in project tez by apache.

the class UnorderedKVInput method start.

@Override
public synchronized void start() throws IOException {
    if (!isStarted.get()) {
        // //// Initial configuration
        memoryUpdateCallbackHandler.validateUpdateReceived();
        CompressionCodec codec;
        if (ConfigUtils.isIntermediateInputCompressed(conf)) {
            Class<? extends CompressionCodec> codecClass = ConfigUtils.getIntermediateInputCompressorClass(conf, DefaultCodec.class);
            codec = ReflectionUtils.newInstance(codecClass, conf);
        } else {
            codec = null;
        }
        boolean compositeFetch = ShuffleUtils.isTezShuffleHandler(conf);
        boolean ifileReadAhead = conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT);
        int ifileReadAheadLength = 0;
        int ifileBufferSize = 0;
        if (ifileReadAhead) {
            ifileReadAheadLength = conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES_DEFAULT);
        }
        ifileBufferSize = conf.getInt("io.file.buffer.size", TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_BUFFER_SIZE_DEFAULT);
        this.inputManager = new SimpleFetchedInputAllocator(TezUtilsInternal.cleanVertexName(getContext().getSourceVertexName()), getContext().getUniqueIdentifier(), getContext().getDagIdentifier(), conf, getContext().getTotalMemoryAvailableToTask(), memoryUpdateCallbackHandler.getMemoryAssigned());
        this.shuffleManager = new ShuffleManager(getContext(), conf, getNumPhysicalInputs(), ifileBufferSize, ifileReadAhead, ifileReadAheadLength, codec, inputManager);
        this.inputEventHandler = new ShuffleInputEventHandlerImpl(getContext(), shuffleManager, inputManager, codec, ifileReadAhead, ifileReadAheadLength, compositeFetch);
        // //// End of Initial configuration
        this.shuffleManager.run();
        this.kvReader = createReader(inputRecordCounter, codec, ifileBufferSize, ifileReadAhead, ifileReadAheadLength);
        List<Event> pending = new LinkedList<Event>();
        pendingEvents.drainTo(pending);
        if (pending.size() > 0) {
            if (LOG.isDebugEnabled()) {
                LOG.debug(getContext().getSourceVertexName() + ": " + "NoAutoStart delay in processing first event: " + (System.currentTimeMillis() - firstEventReceivedTime));
            }
            inputEventHandler.handleEvents(pending);
        }
        isStarted.set(true);
    }
}

Example 2 with ShuffleManager

use of org.apache.tez.runtime.library.common.shuffle.impl.ShuffleManager in project tez by apache.

the class TestUnorderedKVReader method testInterruptOnNext.

@Test(timeout = 5000)
public void testInterruptOnNext() throws IOException, InterruptedException {
    ShuffleManager shuffleManager = mock(ShuffleManager.class);
    // Simulate an interrupt while waiting for the next fetched input.
    doThrow(new InterruptedException()).when(shuffleManager).getNextInput();
    TezCounters counters = new TezCounters();
    TezCounter inputRecords = counters.findCounter(TaskCounter.INPUT_RECORDS_PROCESSED);
    UnorderedKVReader<Text, Text> reader = new UnorderedKVReader<Text, Text>(shuffleManager, defaultConf, null, false, -1, -1, inputRecords, mock(InputContext.class));
    try {
        reader.next();
        fail("No data available to reader. Should not be able to access any record");
    } catch (IOInterruptedException e) {
    // Expected exception. Any other should fail the test.
    }
}

Example 3 with ShuffleManager

use of org.apache.tez.runtime.library.common.shuffle.impl.ShuffleManager in project tez by apache.

the class TestUnorderedKVReader method setupReader.

private void setupReader() throws IOException, InterruptedException {
    defaultConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, Text.class.getName());
    defaultConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, Text.class.getName());
    createIFile(outputPath, 1);
    final LinkedList<LocalDiskFetchedInput> inputs = new LinkedList<LocalDiskFetchedInput>();
    LocalDiskFetchedInput realFetchedInput = new LocalDiskFetchedInput(0, compLen, new InputAttemptIdentifier(0, 0), outputPath, defaultConf, new FetchedInputCallback() {

        @Override
        public void fetchComplete(FetchedInput fetchedInput) {
        }

        @Override
        public void fetchFailed(FetchedInput fetchedInput) {
        }

        @Override
        public void freeResources(FetchedInput fetchedInput) {
        }
    });
    LocalDiskFetchedInput fetchedInput = spy(realFetchedInput);
    doNothing().when(fetchedInput).free();
    inputs.add(fetchedInput);
    TezCounters counters = new TezCounters();
    TezCounter inputRecords = counters.findCounter(TaskCounter.INPUT_RECORDS_PROCESSED);
    ShuffleManager manager = mock(ShuffleManager.class);
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            return (inputs.isEmpty()) ? null : inputs.remove();
        }
    }).when(manager).getNextInput();
    unorderedKVReader = new UnorderedKVReader<Text, Text>(manager, defaultConf, null, false, -1, -1, inputRecords, mock(InputContext.class));
}