Examples with DeserializationDelegate org.apache.flink.runtime.plugable.DeserializationDelegate used on opensource projects

Example 1 with DeserializationDelegate

use of org.apache.flink.runtime.plugable.DeserializationDelegate in project flink by apache.

the class DataSinkTask method initInputReaders.

/**
	 * Initializes the input readers of the DataSinkTask.
	 * 
	 * @throws RuntimeException
	 *         Thrown in case of invalid task input configuration.
	 */
@SuppressWarnings("unchecked")
private void initInputReaders() throws Exception {
    int numGates = 0;
    //  ---------------- create the input readers ---------------------
    // in case where a logical input unions multiple physical inputs, create a union reader
    final int groupSize = this.config.getGroupSize(0);
    numGates += groupSize;
    if (groupSize == 1) {
        // non-union case
        inputReader = new MutableRecordReader<DeserializationDelegate<IT>>(getEnvironment().getInputGate(0), getEnvironment().getTaskManagerInfo().getTmpDirectories());
    } else if (groupSize > 1) {
        // union case
        inputReader = new MutableRecordReader<IOReadableWritable>(new UnionInputGate(getEnvironment().getAllInputGates()), getEnvironment().getTaskManagerInfo().getTmpDirectories());
    } else {
        throw new Exception("Illegal input group size in task configuration: " + groupSize);
    }
    this.inputTypeSerializerFactory = this.config.getInputSerializer(0, getUserCodeClassLoader());
    @SuppressWarnings({ "rawtypes" }) final MutableObjectIterator<?> iter = new ReaderIterator(inputReader, this.inputTypeSerializerFactory.getSerializer());
    this.reader = (MutableObjectIterator<IT>) iter;
    // final sanity check
    if (numGates != this.config.getNumInputs()) {
        throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
    }
}

Example 2 with DeserializationDelegate

use of org.apache.flink.runtime.plugable.DeserializationDelegate in project flink by apache.

the class BroadcastVariableMaterialization method materializeVariable.

// --------------------------------------------------------------------------------------------
public void materializeVariable(MutableReader<?> reader, TypeSerializerFactory<?> serializerFactory, BatchTask<?, ?> referenceHolder) throws MaterializationExpiredException, IOException {
    Preconditions.checkNotNull(reader);
    Preconditions.checkNotNull(serializerFactory);
    Preconditions.checkNotNull(referenceHolder);
    final boolean materializer;
    // that way, other tasks can de-register (in case of failure) while materialization is happening
    synchronized (references) {
        if (disposed) {
            throw new MaterializationExpiredException();
        }
        // sanity check
        if (!references.add(referenceHolder)) {
            throw new IllegalStateException(String.format("The task %s already holds a reference to the broadcast variable %s.", referenceHolder.getEnvironment().getTaskInfo().getTaskNameWithSubtasks(), key.toString()));
        }
        materializer = references.size() == 1;
    }
    try {
        @SuppressWarnings("unchecked") final MutableReader<DeserializationDelegate<T>> typedReader = (MutableReader<DeserializationDelegate<T>>) reader;
        @SuppressWarnings("unchecked") final TypeSerializer<T> serializer = ((TypeSerializerFactory<T>) serializerFactory).getSerializer();
        final ReaderIterator<T> readerIterator = new ReaderIterator<T>(typedReader, serializer);
        if (materializer) {
            // first one, so we need to materialize;
            if (LOG.isDebugEnabled()) {
                LOG.debug("Getting Broadcast Variable (" + key + ") - First access, materializing.");
            }
            ArrayList<T> data = new ArrayList<T>();
            T element;
            while ((element = readerIterator.next()) != null) {
                data.add(element);
            }
            synchronized (materializationMonitor) {
                this.data = data;
                this.materialized = true;
                materializationMonitor.notifyAll();
            }
            if (LOG.isDebugEnabled()) {
                LOG.debug("Materialization of Broadcast Variable (" + key + ") finished.");
            }
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Getting Broadcast Variable (" + key + ") - shared access.");
            }
            T element = serializer.createInstance();
            while ((element = readerIterator.next(element)) != null) ;
            synchronized (materializationMonitor) {
                while (!this.materialized && !disposed) {
                    materializationMonitor.wait();
                }
            }
        }
    } catch (Throwable t) {
        // in case of an exception, we need to clean up big time
        decrementReferenceIfHeld(referenceHolder);
        if (t instanceof IOException) {
            throw (IOException) t;
        } else {
            throw new IOException("Materialization of the broadcast variable failed.", t);
        }
    }
}

Example 3 with DeserializationDelegate

use of org.apache.flink.runtime.plugable.DeserializationDelegate in project flink by apache.

the class StreamMockEnvironment method addOutput.

public <T> void addOutput(final Queue<Object> outputList, final TypeSerializer<T> serializer) {
    try {
        // The record-oriented writers wrap the buffer writer. We mock it
        // to collect the returned buffers and deserialize the content to
        // the output list
        BufferProvider mockBufferProvider = mock(BufferProvider.class);
        when(mockBufferProvider.requestBufferBlocking()).thenAnswer(new Answer<Buffer>() {

            @Override
            public Buffer answer(InvocationOnMock invocationOnMock) throws Throwable {
                return new Buffer(MemorySegmentFactory.allocateUnpooledSegment(bufferSize), mock(BufferRecycler.class));
            }
        });
        ResultPartitionWriter mockWriter = mock(ResultPartitionWriter.class);
        when(mockWriter.getNumberOfOutputChannels()).thenReturn(1);
        when(mockWriter.getBufferProvider()).thenReturn(mockBufferProvider);
        final RecordDeserializer<DeserializationDelegate<T>> recordDeserializer = new AdaptiveSpanningRecordDeserializer<DeserializationDelegate<T>>();
        final NonReusingDeserializationDelegate<T> delegate = new NonReusingDeserializationDelegate<T>(serializer);
        // Add records and events from the buffer to the output list
        doAnswer(new Answer<Void>() {

            @Override
            public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
                Buffer buffer = (Buffer) invocationOnMock.getArguments()[0];
                addBufferToOutputList(recordDeserializer, delegate, buffer, outputList);
                return null;
            }
        }).when(mockWriter).writeBuffer(any(Buffer.class), anyInt());
        doAnswer(new Answer<Void>() {

            @Override
            public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
                Buffer buffer = (Buffer) invocationOnMock.getArguments()[0];
                addBufferToOutputList(recordDeserializer, delegate, buffer, outputList);
                return null;
            }
        }).when(mockWriter).writeBufferToAllChannels(any(Buffer.class));
        outputs.add(mockWriter);
    } catch (Throwable t) {
        t.printStackTrace();
        fail(t.getMessage());
    }
}