Examples with ReaderIterator org.apache.flink.runtime.operators.util.ReaderIterator used on opensource projects

Example 1 with ReaderIterator

use of org.apache.flink.runtime.operators.util.ReaderIterator in project flink by apache.

the class DataSinkTask method initInputReaders.

/**
	 * Initializes the input readers of the DataSinkTask.
	 * 
	 * @throws RuntimeException
	 *         Thrown in case of invalid task input configuration.
	 */
@SuppressWarnings("unchecked")
private void initInputReaders() throws Exception {
    int numGates = 0;
    //  ---------------- create the input readers ---------------------
    // in case where a logical input unions multiple physical inputs, create a union reader
    final int groupSize = this.config.getGroupSize(0);
    numGates += groupSize;
    if (groupSize == 1) {
        // non-union case
        inputReader = new MutableRecordReader<DeserializationDelegate<IT>>(getEnvironment().getInputGate(0), getEnvironment().getTaskManagerInfo().getTmpDirectories());
    } else if (groupSize > 1) {
        // union case
        inputReader = new MutableRecordReader<IOReadableWritable>(new UnionInputGate(getEnvironment().getAllInputGates()), getEnvironment().getTaskManagerInfo().getTmpDirectories());
    } else {
        throw new Exception("Illegal input group size in task configuration: " + groupSize);
    }
    this.inputTypeSerializerFactory = this.config.getInputSerializer(0, getUserCodeClassLoader());
    @SuppressWarnings({ "rawtypes" }) final MutableObjectIterator<?> iter = new ReaderIterator(inputReader, this.inputTypeSerializerFactory.getSerializer());
    this.reader = (MutableObjectIterator<IT>) iter;
    // final sanity check
    if (numGates != this.config.getNumInputs()) {
        throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
    }
}

Example 2 with ReaderIterator

use of org.apache.flink.runtime.operators.util.ReaderIterator in project flink by apache.

the class BroadcastVariableMaterialization method materializeVariable.

// --------------------------------------------------------------------------------------------
public void materializeVariable(MutableReader<?> reader, TypeSerializerFactory<?> serializerFactory, BatchTask<?, ?> referenceHolder) throws MaterializationExpiredException, IOException {
    Preconditions.checkNotNull(reader);
    Preconditions.checkNotNull(serializerFactory);
    Preconditions.checkNotNull(referenceHolder);
    final boolean materializer;
    // that way, other tasks can de-register (in case of failure) while materialization is happening
    synchronized (references) {
        if (disposed) {
            throw new MaterializationExpiredException();
        }
        // sanity check
        if (!references.add(referenceHolder)) {
            throw new IllegalStateException(String.format("The task %s already holds a reference to the broadcast variable %s.", referenceHolder.getEnvironment().getTaskInfo().getTaskNameWithSubtasks(), key.toString()));
        }
        materializer = references.size() == 1;
    }
    try {
        @SuppressWarnings("unchecked") final MutableReader<DeserializationDelegate<T>> typedReader = (MutableReader<DeserializationDelegate<T>>) reader;
        @SuppressWarnings("unchecked") final TypeSerializer<T> serializer = ((TypeSerializerFactory<T>) serializerFactory).getSerializer();
        final ReaderIterator<T> readerIterator = new ReaderIterator<T>(typedReader, serializer);
        if (materializer) {
            // first one, so we need to materialize;
            if (LOG.isDebugEnabled()) {
                LOG.debug("Getting Broadcast Variable (" + key + ") - First access, materializing.");
            }
            ArrayList<T> data = new ArrayList<T>();
            T element;
            while ((element = readerIterator.next()) != null) {
                data.add(element);
            }
            synchronized (materializationMonitor) {
                this.data = data;
                this.materialized = true;
                materializationMonitor.notifyAll();
            }
            if (LOG.isDebugEnabled()) {
                LOG.debug("Materialization of Broadcast Variable (" + key + ") finished.");
            }
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Getting Broadcast Variable (" + key + ") - shared access.");
            }
            T element = serializer.createInstance();
            while ((element = readerIterator.next(element)) != null) ;
            synchronized (materializationMonitor) {
                while (!this.materialized && !disposed) {
                    materializationMonitor.wait();
                }
            }
        }
    } catch (Throwable t) {
        // in case of an exception, we need to clean up big time
        decrementReferenceIfHeld(referenceHolder);
        if (t instanceof IOException) {
            throw (IOException) t;
        } else {
            throw new IOException("Materialization of the broadcast variable failed.", t);
        }
    }
}