Examples with CheckpointBarrier org.apache.flink.runtime.io.network.api.CheckpointBarrier used on opensource projects

Example 6 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class EventSerializer method fromSerializedEvent.

public static AbstractEvent fromSerializedEvent(ByteBuffer buffer, ClassLoader classLoader) throws IOException {
    if (buffer.remaining() < 4) {
        throw new IOException("Incomplete event");
    }
    final ByteOrder bufferOrder = buffer.order();
    buffer.order(ByteOrder.BIG_ENDIAN);
    try {
        int type = buffer.getInt();
        if (type == END_OF_PARTITION_EVENT) {
            return EndOfPartitionEvent.INSTANCE;
        } else if (type == CHECKPOINT_BARRIER_EVENT) {
            long id = buffer.getLong();
            long timestamp = buffer.getLong();
            CheckpointOptions checkpointOptions;
            int checkpointTypeOrdinal = buffer.getInt();
            Preconditions.checkElementIndex(type, CheckpointType.values().length, "Illegal CheckpointType ordinal");
            CheckpointType checkpointType = CheckpointType.values()[checkpointTypeOrdinal];
            if (checkpointType == CheckpointType.FULL_CHECKPOINT) {
                checkpointOptions = CheckpointOptions.forFullCheckpoint();
            } else if (checkpointType == CheckpointType.SAVEPOINT) {
                int len = buffer.getInt();
                byte[] bytes = new byte[len];
                buffer.get(bytes);
                String targetLocation = new String(bytes, STRING_CODING_CHARSET);
                checkpointOptions = CheckpointOptions.forSavepoint(targetLocation);
            } else {
                throw new IOException("Unknown checkpoint type: " + checkpointType);
            }
            return new CheckpointBarrier(id, timestamp, checkpointOptions);
        } else if (type == END_OF_SUPERSTEP_EVENT) {
            return EndOfSuperstepEvent.INSTANCE;
        } else if (type == CANCEL_CHECKPOINT_MARKER_EVENT) {
            long id = buffer.getLong();
            return new CancelCheckpointMarker(id);
        } else if (type == OTHER_EVENT) {
            try {
                final DataInputDeserializer deserializer = new DataInputDeserializer(buffer);
                final String className = deserializer.readUTF();
                final Class<? extends AbstractEvent> clazz;
                try {
                    clazz = classLoader.loadClass(className).asSubclass(AbstractEvent.class);
                } catch (ClassNotFoundException e) {
                    throw new IOException("Could not load event class '" + className + "'.", e);
                } catch (ClassCastException e) {
                    throw new IOException("The class '" + className + "' is not a valid subclass of '" + AbstractEvent.class.getName() + "'.", e);
                }
                final AbstractEvent event = InstantiationUtil.instantiate(clazz, AbstractEvent.class);
                event.read(deserializer);
                return event;
            } catch (Exception e) {
                throw new IOException("Error while deserializing or instantiating event.", e);
            }
        } else {
            throw new IOException("Corrupt byte stream for event");
        }
    } finally {
        buffer.order(bufferOrder);
    }
}

Example 7 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class RecordWriterTest method testBroadcastEventNoRecords.

/**
	 * Tests broadcasting events when no records have been emitted yet.
	 */
@Test
public void testBroadcastEventNoRecords() throws Exception {
    int numChannels = 4;
    int bufferSize = 32;
    @SuppressWarnings("unchecked") Queue<BufferOrEvent>[] queues = new Queue[numChannels];
    for (int i = 0; i < numChannels; i++) {
        queues[i] = new ArrayDeque<>();
    }
    BufferProvider bufferProvider = createBufferProvider(bufferSize);
    ResultPartitionWriter partitionWriter = createCollectingPartitionWriter(queues, bufferProvider);
    RecordWriter<ByteArrayIO> writer = new RecordWriter<>(partitionWriter, new RoundRobin<ByteArrayIO>());
    CheckpointBarrier barrier = new CheckpointBarrier(Integer.MAX_VALUE + 919192L, Integer.MAX_VALUE + 18828228L, CheckpointOptions.forFullCheckpoint());
    // No records emitted yet, broadcast should not request a buffer
    writer.broadcastEvent(barrier);
    verify(bufferProvider, times(0)).requestBufferBlocking();
    for (Queue<BufferOrEvent> queue : queues) {
        assertEquals(1, queue.size());
        BufferOrEvent boe = queue.remove();
        assertTrue(boe.isEvent());
        assertEquals(barrier, boe.getEvent());
    }
}

Example 8 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class EventSerializerTest method testCheckpointBarrierSerialization.

private void testCheckpointBarrierSerialization(long id, long timestamp, CheckpointOptions options) throws IOException {
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    CheckpointBarrier barrier = new CheckpointBarrier(id, timestamp, options);
    ByteBuffer serialized = EventSerializer.toSerializedEvent(barrier);
    CheckpointBarrier deserialized = (CheckpointBarrier) EventSerializer.fromSerializedEvent(serialized, cl);
    assertFalse(serialized.hasRemaining());
    assertEquals(id, deserialized.getId());
    assertEquals(timestamp, deserialized.getTimestamp());
    assertEquals(options.getCheckpointType(), deserialized.getCheckpointOptions().getCheckpointType());
    assertEquals(options.getTargetLocation(), deserialized.getCheckpointOptions().getTargetLocation());
}

Example 9 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class EventSerializerTest method testSerializeDeserializeEvent.

@Test
public void testSerializeDeserializeEvent() throws Exception {
    AbstractEvent[] events = { EndOfPartitionEvent.INSTANCE, EndOfSuperstepEvent.INSTANCE, new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forFullCheckpoint()), new TestTaskEvent(Math.random(), 12361231273L), new CancelCheckpointMarker(287087987329842L) };
    for (AbstractEvent evt : events) {
        ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
        assertTrue(serializedEvent.hasRemaining());
        AbstractEvent deserialized = EventSerializer.fromSerializedEvent(serializedEvent, getClass().getClassLoader());
        assertNotNull(deserialized);
        assertEquals(evt, deserialized);
    }
}

Example 10 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class OneInputStreamTaskTest method testOvertakingCheckpointBarriers.

/**
	 * This test verifies that checkpoint barriers and barrier buffers work correctly with
	 * concurrent checkpoint barriers where one checkpoint is "overtaking" another checkpoint, i.e.
	 * some inputs receive barriers from an earlier checkpoint, thereby blocking,
	 * then all inputs receive barriers from a later checkpoint.
	 */
@Test
public void testOvertakingCheckpointBarriers() throws Exception {
    final OneInputStreamTask<String, String> mapTask = new OneInputStreamTask<String, String>();
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<String, String>(mapTask, 2, 2, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    StreamMap<String, String> mapOperator = new StreamMap<String, String>(new IdentityMap());
    streamConfig.setStreamOperator(mapOperator);
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
    long initialTime = 0L;
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 0, 0);
    // These elements should be buffered until we receive barriers from
    // all inputs
    testHarness.processElement(new StreamRecord<String>("Hello-0-0", initialTime), 0, 0);
    testHarness.processElement(new StreamRecord<String>("Ciao-0-0", initialTime), 0, 0);
    // These elements should be forwarded, since we did not yet receive a checkpoint barrier
    // on that input, only add to same input, otherwise we would not know the ordering
    // of the output since the Task might read the inputs in any order
    testHarness.processElement(new StreamRecord<String>("Hello-1-1", initialTime), 1, 1);
    testHarness.processElement(new StreamRecord<String>("Ciao-1-1", initialTime), 1, 1);
    expectedOutput.add(new StreamRecord<String>("Hello-1-1", initialTime));
    expectedOutput.add(new StreamRecord<String>("Ciao-1-1", initialTime));
    testHarness.waitForInputProcessing();
    // we should not yet see the barrier, only the two elements from non-blocked input
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // Now give a later barrier to all inputs, this should unblock the first channel,
    // thereby allowing the two blocked elements through
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forFullCheckpoint()), 0, 0);
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forFullCheckpoint()), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forFullCheckpoint()), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forFullCheckpoint()), 1, 1);
    expectedOutput.add(new CancelCheckpointMarker(0));
    expectedOutput.add(new StreamRecord<String>("Hello-0-0", initialTime));
    expectedOutput.add(new StreamRecord<String>("Ciao-0-0", initialTime));
    expectedOutput.add(new CheckpointBarrier(1, 1, CheckpointOptions.forFullCheckpoint()));
    testHarness.waitForInputProcessing();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // Then give the earlier barrier, these should be ignored
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 1, 1);
    testHarness.waitForInputProcessing();
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}