Examples with BufferOrEvent org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent used on opensource projects

Example 1 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class StreamInputProcessor method processInput.

public boolean processInput() throws Exception {
    if (isFinished) {
        return false;
    }
    if (numRecordsIn == null) {
        numRecordsIn = ((OperatorMetricGroup) streamOperator.getMetricGroup()).getIOMetricGroup().getNumRecordsInCounter();
    }
    while (true) {
        if (currentRecordDeserializer != null) {
            DeserializationResult result = currentRecordDeserializer.getNextRecord(deserializationDelegate);
            if (result.isBufferConsumed()) {
                currentRecordDeserializer.getCurrentBuffer().recycle();
                currentRecordDeserializer = null;
            }
            if (result.isFullRecord()) {
                StreamElement recordOrMark = deserializationDelegate.getInstance();
                if (recordOrMark.isWatermark()) {
                    // handle watermark
                    statusWatermarkValve.inputWatermark(recordOrMark.asWatermark(), currentChannel);
                    continue;
                } else if (recordOrMark.isStreamStatus()) {
                    // handle stream status
                    statusWatermarkValve.inputStreamStatus(recordOrMark.asStreamStatus(), currentChannel);
                    continue;
                } else if (recordOrMark.isLatencyMarker()) {
                    // handle latency marker
                    synchronized (lock) {
                        streamOperator.processLatencyMarker(recordOrMark.asLatencyMarker());
                    }
                    continue;
                } else {
                    // now we can do the actual processing
                    StreamRecord<IN> record = recordOrMark.asRecord();
                    synchronized (lock) {
                        numRecordsIn.inc();
                        streamOperator.setKeyContextElement1(record);
                        streamOperator.processElement(record);
                    }
                    return true;
                }
            }
        }
        final BufferOrEvent bufferOrEvent = barrierHandler.getNextNonBlocked();
        if (bufferOrEvent != null) {
            if (bufferOrEvent.isBuffer()) {
                currentChannel = bufferOrEvent.getChannelIndex();
                currentRecordDeserializer = recordDeserializers[currentChannel];
                currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer());
            } else {
                // Event received
                final AbstractEvent event = bufferOrEvent.getEvent();
                if (event.getClass() != EndOfPartitionEvent.class) {
                    throw new IOException("Unexpected event: " + event);
                }
            }
        } else {
            isFinished = true;
            if (!barrierHandler.isEmpty()) {
                throw new IllegalStateException("Trailing data in checkpoint barrier handler.");
            }
            return false;
        }
    }
}

Example 2 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class StreamTwoInputProcessor method processInput.

public boolean processInput() throws Exception {
    if (isFinished) {
        return false;
    }
    while (true) {
        if (currentRecordDeserializer != null) {
            DeserializationResult result;
            if (currentChannel < numInputChannels1) {
                result = currentRecordDeserializer.getNextRecord(deserializationDelegate1);
            } else {
                result = currentRecordDeserializer.getNextRecord(deserializationDelegate2);
            }
            if (result.isBufferConsumed()) {
                currentRecordDeserializer.getCurrentBuffer().recycle();
                currentRecordDeserializer = null;
            }
            if (result.isFullRecord()) {
                if (currentChannel < numInputChannels1) {
                    StreamElement recordOrWatermark = deserializationDelegate1.getInstance();
                    if (recordOrWatermark.isWatermark()) {
                        statusWatermarkValve1.inputWatermark(recordOrWatermark.asWatermark(), currentChannel);
                        continue;
                    } else if (recordOrWatermark.isStreamStatus()) {
                        statusWatermarkValve1.inputStreamStatus(recordOrWatermark.asStreamStatus(), currentChannel);
                        continue;
                    } else if (recordOrWatermark.isLatencyMarker()) {
                        synchronized (lock) {
                            streamOperator.processLatencyMarker1(recordOrWatermark.asLatencyMarker());
                        }
                        continue;
                    } else {
                        StreamRecord<IN1> record = recordOrWatermark.asRecord();
                        synchronized (lock) {
                            streamOperator.setKeyContextElement1(record);
                            streamOperator.processElement1(record);
                        }
                        return true;
                    }
                } else {
                    StreamElement recordOrWatermark = deserializationDelegate2.getInstance();
                    if (recordOrWatermark.isWatermark()) {
                        statusWatermarkValve2.inputWatermark(recordOrWatermark.asWatermark(), currentChannel - numInputChannels1);
                        continue;
                    } else if (recordOrWatermark.isStreamStatus()) {
                        statusWatermarkValve2.inputStreamStatus(recordOrWatermark.asStreamStatus(), currentChannel - numInputChannels1);
                        continue;
                    } else if (recordOrWatermark.isLatencyMarker()) {
                        synchronized (lock) {
                            streamOperator.processLatencyMarker2(recordOrWatermark.asLatencyMarker());
                        }
                        continue;
                    } else {
                        StreamRecord<IN2> record = recordOrWatermark.asRecord();
                        synchronized (lock) {
                            streamOperator.setKeyContextElement2(record);
                            streamOperator.processElement2(record);
                        }
                        return true;
                    }
                }
            }
        }
        final BufferOrEvent bufferOrEvent = barrierHandler.getNextNonBlocked();
        if (bufferOrEvent != null) {
            if (bufferOrEvent.isBuffer()) {
                currentChannel = bufferOrEvent.getChannelIndex();
                currentRecordDeserializer = recordDeserializers[currentChannel];
                currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer());
            } else {
                // Event received
                final AbstractEvent event = bufferOrEvent.getEvent();
                if (event.getClass() != EndOfPartitionEvent.class) {
                    throw new IOException("Unexpected event: " + event);
                }
            }
        } else {
            isFinished = true;
            if (!barrierHandler.isEmpty()) {
                throw new IllegalStateException("Trailing data in checkpoint barrier handler.");
            }
            return false;
        }
    }
}

Example 3 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierBufferMassiveRandomTest method testWithTwoChannelsAndRandomBarriers.

@Test
public void testWithTwoChannelsAndRandomBarriers() {
    IOManager ioMan = null;
    try {
        ioMan = new IOManagerAsync();
        BufferPool pool1 = new NetworkBufferPool(100, PAGE_SIZE, MemoryType.HEAP).createBufferPool(100, 100);
        BufferPool pool2 = new NetworkBufferPool(100, PAGE_SIZE, MemoryType.HEAP).createBufferPool(100, 100);
        RandomGeneratingInputGate myIG = new RandomGeneratingInputGate(new BufferPool[] { pool1, pool2 }, new BarrierGenerator[] { new CountBarrier(100000), new RandomBarrier(100000) });
        BarrierBuffer barrierBuffer = new BarrierBuffer(myIG, ioMan);
        for (int i = 0; i < 2000000; i++) {
            BufferOrEvent boe = barrierBuffer.getNextNonBlocked();
            if (boe.isBuffer()) {
                boe.getBuffer().recycle();
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        if (ioMan != null) {
            ioMan.shutdown();
        }
    }
}

Example 4 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierBufferTest method testMultiChannelAbortCheckpoint.

@Test
public void testMultiChannelAbortCheckpoint() throws Exception {
    BufferOrEvent[] sequence = { /* 0 */
    createBuffer(0), createBuffer(2), createBuffer(0), /* 3 */
    createBarrier(1, 1), createBarrier(1, 2), /* 5 */
    createBuffer(2), createBuffer(1), /* 7 */
    createBarrier(1, 0), /* 8 */
    createBuffer(0), createBuffer(2), /* 10 */
    createBarrier(2, 0), createBarrier(2, 2), /* 12 */
    createBuffer(0), createBuffer(2), /* 14 */
    createCancellationBarrier(2, 1), /* 15 */
    createBuffer(2), createBuffer(1), /* 17 */
    createBarrier(3, 1), createBarrier(3, 2), createBarrier(3, 0), /* 20 */
    createBuffer(0), createBuffer(1), /* 22 */
    createCancellationBarrier(4, 1), createBarrier(4, 2), /* 24 */
    createBuffer(0), /* 25 */
    createBarrier(4, 0), /* 26 */
    createBuffer(0), createBuffer(1), createBuffer(2), /* 29 */
    createBarrier(5, 2), createBarrier(5, 1), createBarrier(5, 0), /* 32 */
    createBuffer(0), createBuffer(1), /* 34 */
    createCancellationBarrier(6, 1), createCancellationBarrier(6, 2), /* 36 */
    createBarrier(6, 0), /* 37 */
    createBuffer(0) };
    MockInputGate gate = new MockInputGate(PAGE_SIZE, 3, Arrays.asList(sequence));
    BarrierBuffer buffer = new BarrierBuffer(gate, IO_MANAGER);
    StatefulTask toNotify = mock(StatefulTask.class);
    buffer.registerCheckpointEventHandler(toNotify);
    long startTs;
    // successful first checkpoint, with some aligned buffers
    check(sequence[0], buffer.getNextNonBlocked());
    check(sequence[1], buffer.getNextNonBlocked());
    check(sequence[2], buffer.getNextNonBlocked());
    startTs = System.nanoTime();
    check(sequence[5], buffer.getNextNonBlocked());
    verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(1L)), any(CheckpointOptions.class), any(CheckpointMetrics.class));
    validateAlignmentTime(startTs, buffer);
    check(sequence[6], buffer.getNextNonBlocked());
    check(sequence[8], buffer.getNextNonBlocked());
    check(sequence[9], buffer.getNextNonBlocked());
    // canceled checkpoint on last barrier
    startTs = System.nanoTime();
    check(sequence[12], buffer.getNextNonBlocked());
    verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(2L), any(CheckpointDeclineOnCancellationBarrierException.class));
    validateAlignmentTime(startTs, buffer);
    check(sequence[13], buffer.getNextNonBlocked());
    // one more successful checkpoint
    check(sequence[15], buffer.getNextNonBlocked());
    check(sequence[16], buffer.getNextNonBlocked());
    startTs = System.nanoTime();
    check(sequence[20], buffer.getNextNonBlocked());
    verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(3L)), any(CheckpointOptions.class), any(CheckpointMetrics.class));
    validateAlignmentTime(startTs, buffer);
    check(sequence[21], buffer.getNextNonBlocked());
    // this checkpoint gets immediately canceled
    check(sequence[24], buffer.getNextNonBlocked());
    verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(4L), any(CheckpointDeclineOnCancellationBarrierException.class));
    assertEquals(0L, buffer.getAlignmentDurationNanos());
    // some buffers
    check(sequence[26], buffer.getNextNonBlocked());
    check(sequence[27], buffer.getNextNonBlocked());
    check(sequence[28], buffer.getNextNonBlocked());
    // a simple successful checkpoint
    startTs = System.nanoTime();
    check(sequence[32], buffer.getNextNonBlocked());
    verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(5L)), any(CheckpointOptions.class), any(CheckpointMetrics.class));
    validateAlignmentTime(startTs, buffer);
    check(sequence[33], buffer.getNextNonBlocked());
    check(sequence[37], buffer.getNextNonBlocked());
    verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(6L), any(CheckpointDeclineOnCancellationBarrierException.class));
    assertEquals(0L, buffer.getAlignmentDurationNanos());
    // all done
    assertNull(buffer.getNextNonBlocked());
    assertNull(buffer.getNextNonBlocked());
    buffer.cleanup();
    checkNoTempFilesRemain();
}

Example 5 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierBufferTest method testSingleChannelWithBarriers.

/**
	 * Validates that the buffer preserved the order of elements for a 
	 * input with a single input channel, and checkpoint events.
	 */
@Test
public void testSingleChannelWithBarriers() {
    try {
        BufferOrEvent[] sequence = { createBuffer(0), createBuffer(0), createBuffer(0), createBarrier(1, 0), createBuffer(0), createBuffer(0), createBuffer(0), createBuffer(0), createBarrier(2, 0), createBarrier(3, 0), createBuffer(0), createBuffer(0), createBarrier(4, 0), createBarrier(5, 0), createBarrier(6, 0), createBuffer(0), createEndOfPartition(0) };
        MockInputGate gate = new MockInputGate(PAGE_SIZE, 1, Arrays.asList(sequence));
        BarrierBuffer buffer = new BarrierBuffer(gate, IO_MANAGER);
        ValidatingCheckpointHandler handler = new ValidatingCheckpointHandler();
        buffer.registerCheckpointEventHandler(handler);
        handler.setNextExpectedCheckpointId(1L);
        for (BufferOrEvent boe : sequence) {
            if (boe.isBuffer() || boe.getEvent().getClass() != CheckpointBarrier.class) {
                assertEquals(boe, buffer.getNextNonBlocked());
            }
        }
        assertNull(buffer.getNextNonBlocked());
        assertNull(buffer.getNextNonBlocked());
        buffer.cleanup();
        checkNoTempFilesRemain();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}