Examples with MutableReader org.apache.flink.runtime.io.network.api.reader.MutableReader used on opensource projects

Example 1 with MutableReader

use of org.apache.flink.runtime.io.network.api.reader.MutableReader in project flink by apache.

the class BatchTask method initInputReaders.

/**
	 * Creates the record readers for the number of inputs as defined by {@link #getNumTaskInputs()}.
	 *
	 * This method requires that the task configuration, the driver, and the user-code class loader are set.
	 */
protected void initInputReaders() throws Exception {
    final int numInputs = getNumTaskInputs();
    final MutableReader<?>[] inputReaders = new MutableReader<?>[numInputs];
    int currentReaderOffset = 0;
    for (int i = 0; i < numInputs; i++) {
        //  ---------------- create the input readers ---------------------
        // in case where a logical input unions multiple physical inputs, create a union reader
        final int groupSize = this.config.getGroupSize(i);
        if (groupSize == 1) {
            // non-union case
            inputReaders[i] = new MutableRecordReader<IOReadableWritable>(getEnvironment().getInputGate(currentReaderOffset), getEnvironment().getTaskManagerInfo().getTmpDirectories());
        } else if (groupSize > 1) {
            // union case
            InputGate[] readers = new InputGate[groupSize];
            for (int j = 0; j < groupSize; ++j) {
                readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
            }
            inputReaders[i] = new MutableRecordReader<IOReadableWritable>(new UnionInputGate(readers), getEnvironment().getTaskManagerInfo().getTmpDirectories());
        } else {
            throw new Exception("Illegal input group size in task configuration: " + groupSize);
        }
        currentReaderOffset += groupSize;
    }
    this.inputReaders = inputReaders;
    // final sanity check
    if (currentReaderOffset != this.config.getNumInputs()) {
        throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
    }
}

Example 2 with MutableReader

use of org.apache.flink.runtime.io.network.api.reader.MutableReader in project flink by apache.

the class BroadcastVariableMaterialization method materializeVariable.

// --------------------------------------------------------------------------------------------
public void materializeVariable(MutableReader<?> reader, TypeSerializerFactory<?> serializerFactory, BatchTask<?, ?> referenceHolder) throws MaterializationExpiredException, IOException {
    Preconditions.checkNotNull(reader);
    Preconditions.checkNotNull(serializerFactory);
    Preconditions.checkNotNull(referenceHolder);
    final boolean materializer;
    // that way, other tasks can de-register (in case of failure) while materialization is happening
    synchronized (references) {
        if (disposed) {
            throw new MaterializationExpiredException();
        }
        // sanity check
        if (!references.add(referenceHolder)) {
            throw new IllegalStateException(String.format("The task %s already holds a reference to the broadcast variable %s.", referenceHolder.getEnvironment().getTaskInfo().getTaskNameWithSubtasks(), key.toString()));
        }
        materializer = references.size() == 1;
    }
    try {
        @SuppressWarnings("unchecked") final MutableReader<DeserializationDelegate<T>> typedReader = (MutableReader<DeserializationDelegate<T>>) reader;
        @SuppressWarnings("unchecked") final TypeSerializer<T> serializer = ((TypeSerializerFactory<T>) serializerFactory).getSerializer();
        final ReaderIterator<T> readerIterator = new ReaderIterator<T>(typedReader, serializer);
        if (materializer) {
            // first one, so we need to materialize;
            if (LOG.isDebugEnabled()) {
                LOG.debug("Getting Broadcast Variable (" + key + ") - First access, materializing.");
            }
            ArrayList<T> data = new ArrayList<T>();
            T element;
            while ((element = readerIterator.next()) != null) {
                data.add(element);
            }
            synchronized (materializationMonitor) {
                this.data = data;
                this.materialized = true;
                materializationMonitor.notifyAll();
            }
            if (LOG.isDebugEnabled()) {
                LOG.debug("Materialization of Broadcast Variable (" + key + ") finished.");
            }
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Getting Broadcast Variable (" + key + ") - shared access.");
            }
            T element = serializer.createInstance();
            while ((element = readerIterator.next(element)) != null) ;
            synchronized (materializationMonitor) {
                while (!this.materialized && !disposed) {
                    materializationMonitor.wait();
                }
            }
        }
    } catch (Throwable t) {
        // in case of an exception, we need to clean up big time
        decrementReferenceIfHeld(referenceHolder);
        if (t instanceof IOException) {
            throw (IOException) t;
        } else {
            throw new IOException("Materialization of the broadcast variable failed.", t);
        }
    }
}

Example 3 with MutableReader

use of org.apache.flink.runtime.io.network.api.reader.MutableReader in project flink by apache.

the class BatchTask method initBroadcastInputReaders.

/**
	 * Creates the record readers for the extra broadcast inputs as configured by {@link TaskConfig#getNumBroadcastInputs()}.
	 *
	 * This method requires that the task configuration, the driver, and the user-code class loader are set.
	 */
protected void initBroadcastInputReaders() throws Exception {
    final int numBroadcastInputs = this.config.getNumBroadcastInputs();
    final MutableReader<?>[] broadcastInputReaders = new MutableReader<?>[numBroadcastInputs];
    int currentReaderOffset = config.getNumInputs();
    for (int i = 0; i < this.config.getNumBroadcastInputs(); i++) {
        //  ---------------- create the input readers ---------------------
        // in case where a logical input unions multiple physical inputs, create a union reader
        final int groupSize = this.config.getBroadcastGroupSize(i);
        if (groupSize == 1) {
            // non-union case
            broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(getEnvironment().getInputGate(currentReaderOffset), getEnvironment().getTaskManagerInfo().getTmpDirectories());
        } else if (groupSize > 1) {
            // union case
            InputGate[] readers = new InputGate[groupSize];
            for (int j = 0; j < groupSize; ++j) {
                readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
            }
            broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(new UnionInputGate(readers), getEnvironment().getTaskManagerInfo().getTmpDirectories());
        } else {
            throw new Exception("Illegal input group size in task configuration: " + groupSize);
        }
        currentReaderOffset += groupSize;
    }
    this.broadcastInputReaders = broadcastInputReaders;
}