Examples with UnionInputGate org.apache.flink.runtime.io.network.partition.consumer.UnionInputGate used on opensource projects

Example 1 with UnionInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.UnionInputGate in project flink by apache.

the class BatchTask method initInputReaders.

/**
	 * Creates the record readers for the number of inputs as defined by {@link #getNumTaskInputs()}.
	 *
	 * This method requires that the task configuration, the driver, and the user-code class loader are set.
	 */
protected void initInputReaders() throws Exception {
    final int numInputs = getNumTaskInputs();
    final MutableReader<?>[] inputReaders = new MutableReader<?>[numInputs];
    int currentReaderOffset = 0;
    for (int i = 0; i < numInputs; i++) {
        //  ---------------- create the input readers ---------------------
        // in case where a logical input unions multiple physical inputs, create a union reader
        final int groupSize = this.config.getGroupSize(i);
        if (groupSize == 1) {
            // non-union case
            inputReaders[i] = new MutableRecordReader<IOReadableWritable>(getEnvironment().getInputGate(currentReaderOffset), getEnvironment().getTaskManagerInfo().getTmpDirectories());
        } else if (groupSize > 1) {
            // union case
            InputGate[] readers = new InputGate[groupSize];
            for (int j = 0; j < groupSize; ++j) {
                readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
            }
            inputReaders[i] = new MutableRecordReader<IOReadableWritable>(new UnionInputGate(readers), getEnvironment().getTaskManagerInfo().getTmpDirectories());
        } else {
            throw new Exception("Illegal input group size in task configuration: " + groupSize);
        }
        currentReaderOffset += groupSize;
    }
    this.inputReaders = inputReaders;
    // final sanity check
    if (currentReaderOffset != this.config.getNumInputs()) {
        throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
    }
}

Example 2 with UnionInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.UnionInputGate in project flink by apache.

the class DataSinkTask method initInputReaders.

/**
	 * Initializes the input readers of the DataSinkTask.
	 * 
	 * @throws RuntimeException
	 *         Thrown in case of invalid task input configuration.
	 */
@SuppressWarnings("unchecked")
private void initInputReaders() throws Exception {
    int numGates = 0;
    //  ---------------- create the input readers ---------------------
    // in case where a logical input unions multiple physical inputs, create a union reader
    final int groupSize = this.config.getGroupSize(0);
    numGates += groupSize;
    if (groupSize == 1) {
        // non-union case
        inputReader = new MutableRecordReader<DeserializationDelegate<IT>>(getEnvironment().getInputGate(0), getEnvironment().getTaskManagerInfo().getTmpDirectories());
    } else if (groupSize > 1) {
        // union case
        inputReader = new MutableRecordReader<IOReadableWritable>(new UnionInputGate(getEnvironment().getAllInputGates()), getEnvironment().getTaskManagerInfo().getTmpDirectories());
    } else {
        throw new Exception("Illegal input group size in task configuration: " + groupSize);
    }
    this.inputTypeSerializerFactory = this.config.getInputSerializer(0, getUserCodeClassLoader());
    @SuppressWarnings({ "rawtypes" }) final MutableObjectIterator<?> iter = new ReaderIterator(inputReader, this.inputTypeSerializerFactory.getSerializer());
    this.reader = (MutableObjectIterator<IT>) iter;
    // final sanity check
    if (numGates != this.config.getNumInputs()) {
        throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
    }
}

Example 3 with UnionInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.UnionInputGate in project flink by apache.

the class BatchTask method initBroadcastInputReaders.

/**
	 * Creates the record readers for the extra broadcast inputs as configured by {@link TaskConfig#getNumBroadcastInputs()}.
	 *
	 * This method requires that the task configuration, the driver, and the user-code class loader are set.
	 */
protected void initBroadcastInputReaders() throws Exception {
    final int numBroadcastInputs = this.config.getNumBroadcastInputs();
    final MutableReader<?>[] broadcastInputReaders = new MutableReader<?>[numBroadcastInputs];
    int currentReaderOffset = config.getNumInputs();
    for (int i = 0; i < this.config.getNumBroadcastInputs(); i++) {
        //  ---------------- create the input readers ---------------------
        // in case where a logical input unions multiple physical inputs, create a union reader
        final int groupSize = this.config.getBroadcastGroupSize(i);
        if (groupSize == 1) {
            // non-union case
            broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(getEnvironment().getInputGate(currentReaderOffset), getEnvironment().getTaskManagerInfo().getTmpDirectories());
        } else if (groupSize > 1) {
            // union case
            InputGate[] readers = new InputGate[groupSize];
            for (int j = 0; j < groupSize; ++j) {
                readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
            }
            broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(new UnionInputGate(readers), getEnvironment().getTaskManagerInfo().getTmpDirectories());
        } else {
            throw new Exception("Illegal input group size in task configuration: " + groupSize);
        }
        currentReaderOffset += groupSize;
    }
    this.broadcastInputReaders = broadcastInputReaders;
}