Examples with SimpleFetchedInputAllocator org.apache.tez.runtime.library.common.shuffle.impl.SimpleFetchedInputAllocator used on opensource projects

Example 1 with SimpleFetchedInputAllocator

use of org.apache.tez.runtime.library.common.shuffle.impl.SimpleFetchedInputAllocator in project tez by apache.

the class UnorderedKVInput method start.

@Override
public synchronized void start() throws IOException {
    if (!isStarted.get()) {
        // //// Initial configuration
        memoryUpdateCallbackHandler.validateUpdateReceived();
        CompressionCodec codec;
        if (ConfigUtils.isIntermediateInputCompressed(conf)) {
            Class<? extends CompressionCodec> codecClass = ConfigUtils.getIntermediateInputCompressorClass(conf, DefaultCodec.class);
            codec = ReflectionUtils.newInstance(codecClass, conf);
        } else {
            codec = null;
        }
        boolean compositeFetch = ShuffleUtils.isTezShuffleHandler(conf);
        boolean ifileReadAhead = conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT);
        int ifileReadAheadLength = 0;
        int ifileBufferSize = 0;
        if (ifileReadAhead) {
            ifileReadAheadLength = conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES_DEFAULT);
        }
        ifileBufferSize = conf.getInt("io.file.buffer.size", TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_BUFFER_SIZE_DEFAULT);
        this.inputManager = new SimpleFetchedInputAllocator(TezUtilsInternal.cleanVertexName(getContext().getSourceVertexName()), getContext().getUniqueIdentifier(), getContext().getDagIdentifier(), conf, getContext().getTotalMemoryAvailableToTask(), memoryUpdateCallbackHandler.getMemoryAssigned());
        this.shuffleManager = new ShuffleManager(getContext(), conf, getNumPhysicalInputs(), ifileBufferSize, ifileReadAhead, ifileReadAheadLength, codec, inputManager);
        this.inputEventHandler = new ShuffleInputEventHandlerImpl(getContext(), shuffleManager, inputManager, codec, ifileReadAhead, ifileReadAheadLength, compositeFetch);
        // //// End of Initial configuration
        this.shuffleManager.run();
        this.kvReader = createReader(inputRecordCounter, codec, ifileBufferSize, ifileReadAhead, ifileReadAheadLength);
        List<Event> pending = new LinkedList<Event>();
        pendingEvents.drainTo(pending);
        if (pending.size() > 0) {
            if (LOG.isDebugEnabled()) {
                LOG.debug(getContext().getSourceVertexName() + ": " + "NoAutoStart delay in processing first event: " + (System.currentTimeMillis() - firstEventReceivedTime));
            }
            inputEventHandler.handleEvents(pending);
        }
        isStarted.set(true);
    }
}