Examples with CancelCheckpointMarker org.apache.flink.runtime.io.network.api.CancelCheckpointMarker used on opensource projects

Example 1 with CancelCheckpointMarker

use of org.apache.flink.runtime.io.network.api.CancelCheckpointMarker in project flink by apache.

the class TwoInputStreamTaskTest method testOvertakingCheckpointBarriers.

/**
	 * This test verifies that checkpoint barriers and barrier buffers work correctly with
	 * concurrent checkpoint barriers where one checkpoint is "overtaking" another checkpoint, i.e.
	 * some inputs receive barriers from an earlier checkpoint, thereby blocking,
	 * then all inputs receive barriers from a later checkpoint.
	 */
@Test
@SuppressWarnings("unchecked")
public void testOvertakingCheckpointBarriers() throws Exception {
    final TwoInputStreamTask<String, Integer, String> coMapTask = new TwoInputStreamTask<String, Integer, String>();
    final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness = new TwoInputStreamTaskTestHarness<String, Integer, String>(coMapTask, 2, 2, new int[] { 1, 2 }, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    CoStreamMap<String, Integer, String> coMapOperator = new CoStreamMap<String, Integer, String>(new IdentityMap());
    streamConfig.setStreamOperator(coMapOperator);
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
    long initialTime = 0L;
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 0, 0);
    // These elements should be buffered until we receive barriers from
    // all inputs
    testHarness.processElement(new StreamRecord<String>("Hello-0-0", initialTime), 0, 0);
    testHarness.processElement(new StreamRecord<String>("Ciao-0-0", initialTime), 0, 0);
    // These elements should be forwarded, since we did not yet receive a checkpoint barrier
    // on that input, only add to same input, otherwise we would not know the ordering
    // of the output since the Task might read the inputs in any order
    testHarness.processElement(new StreamRecord<Integer>(42, initialTime), 1, 1);
    testHarness.processElement(new StreamRecord<Integer>(1337, initialTime), 1, 1);
    expectedOutput.add(new StreamRecord<String>("42", initialTime));
    expectedOutput.add(new StreamRecord<String>("1337", initialTime));
    testHarness.waitForInputProcessing();
    // we should not yet see the barrier, only the two elements from non-blocked input
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // Now give a later barrier to all inputs, this should unblock the first channel,
    // thereby allowing the two blocked elements through
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forFullCheckpoint()), 0, 0);
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forFullCheckpoint()), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forFullCheckpoint()), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forFullCheckpoint()), 1, 1);
    expectedOutput.add(new CancelCheckpointMarker(0));
    expectedOutput.add(new StreamRecord<String>("Hello-0-0", initialTime));
    expectedOutput.add(new StreamRecord<String>("Ciao-0-0", initialTime));
    expectedOutput.add(new CheckpointBarrier(1, 1, CheckpointOptions.forFullCheckpoint()));
    testHarness.waitForInputProcessing();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // Then give the earlier barrier, these should be ignored
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forFullCheckpoint()), 1, 1);
    testHarness.waitForInputProcessing();
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}

Example 2 with CancelCheckpointMarker

use of org.apache.flink.runtime.io.network.api.CancelCheckpointMarker in project flink by apache.

the class StreamTaskCancellationBarrierTest method testDeclineCallOnCancelBarrierTwoInputs.

/**
	 * This test verifies (for onw input tasks) that the Stream tasks react the following way to
	 * receiving a checkpoint cancellation barrier:
	 *
	 *   - send a "decline checkpoint" notification out (to the JobManager)
	 *   - emit a cancellation barrier downstream
	 */
@Test
public void testDeclineCallOnCancelBarrierTwoInputs() throws Exception {
    TwoInputStreamTask<String, String, String> task = new TwoInputStreamTask<String, String, String>();
    TwoInputStreamTaskTestHarness<String, String, String> testHarness = new TwoInputStreamTaskTestHarness<>(task, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    CoStreamMap<String, String, String> op = new CoStreamMap<>(new UnionCoMap());
    streamConfig.setStreamOperator(op);
    StreamMockEnvironment environment = spy(testHarness.createEnvironment());
    // start the task
    testHarness.invoke(environment);
    testHarness.waitForTaskRunning();
    // emit cancellation barriers
    testHarness.processEvent(new CancelCheckpointMarker(2L), 0, 0);
    testHarness.processEvent(new CancelCheckpointMarker(2L), 1, 0);
    testHarness.waitForInputProcessing();
    // the decline call should go to the coordinator
    verify(environment, times(1)).declineCheckpoint(eq(2L), any(CheckpointDeclineOnCancellationBarrierException.class));
    // a cancellation barrier should be downstream
    Object result = testHarness.getOutput().poll();
    assertNotNull("nothing emitted", result);
    assertTrue("wrong type emitted", result instanceof CancelCheckpointMarker);
    assertEquals("wrong checkpoint id", 2L, ((CancelCheckpointMarker) result).getCheckpointId());
    // cancel and shutdown
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
}

Example 3 with CancelCheckpointMarker

use of org.apache.flink.runtime.io.network.api.CancelCheckpointMarker in project flink by apache.

the class EventSerializer method toSerializedEvent.

// ------------------------------------------------------------------------
public static ByteBuffer toSerializedEvent(AbstractEvent event) throws IOException {
    final Class<?> eventClass = event.getClass();
    if (eventClass == EndOfPartitionEvent.class) {
        return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_PARTITION_EVENT });
    } else if (eventClass == CheckpointBarrier.class) {
        CheckpointBarrier barrier = (CheckpointBarrier) event;
        CheckpointOptions checkpointOptions = barrier.getCheckpointOptions();
        CheckpointType checkpointType = checkpointOptions.getCheckpointType();
        ByteBuffer buf;
        if (checkpointType == CheckpointType.FULL_CHECKPOINT) {
            buf = ByteBuffer.allocate(24);
            buf.putInt(0, CHECKPOINT_BARRIER_EVENT);
            buf.putLong(4, barrier.getId());
            buf.putLong(12, barrier.getTimestamp());
            buf.putInt(20, checkpointType.ordinal());
        } else if (checkpointType == CheckpointType.SAVEPOINT) {
            String targetLocation = checkpointOptions.getTargetLocation();
            assert (targetLocation != null);
            byte[] locationBytes = targetLocation.getBytes(STRING_CODING_CHARSET);
            buf = ByteBuffer.allocate(24 + 4 + locationBytes.length);
            buf.putInt(0, CHECKPOINT_BARRIER_EVENT);
            buf.putLong(4, barrier.getId());
            buf.putLong(12, barrier.getTimestamp());
            buf.putInt(20, checkpointType.ordinal());
            buf.putInt(24, locationBytes.length);
            for (int i = 0; i < locationBytes.length; i++) {
                buf.put(28 + i, locationBytes[i]);
            }
        } else {
            throw new IOException("Unknown checkpoint type: " + checkpointType);
        }
        return buf;
    } else if (eventClass == EndOfSuperstepEvent.class) {
        return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_SUPERSTEP_EVENT });
    } else if (eventClass == CancelCheckpointMarker.class) {
        CancelCheckpointMarker marker = (CancelCheckpointMarker) event;
        ByteBuffer buf = ByteBuffer.allocate(12);
        buf.putInt(0, CANCEL_CHECKPOINT_MARKER_EVENT);
        buf.putLong(4, marker.getCheckpointId());
        return buf;
    } else {
        try {
            final DataOutputSerializer serializer = new DataOutputSerializer(128);
            serializer.writeInt(OTHER_EVENT);
            serializer.writeUTF(event.getClass().getName());
            event.write(serializer);
            return serializer.wrapAsByteBuffer();
        } catch (IOException e) {
            throw new IOException("Error while serializing event.", e);
        }
    }
}

Example 4 with CancelCheckpointMarker

use of org.apache.flink.runtime.io.network.api.CancelCheckpointMarker in project flink by apache.

the class EventSerializer method fromSerializedEvent.

public static AbstractEvent fromSerializedEvent(ByteBuffer buffer, ClassLoader classLoader) throws IOException {
    if (buffer.remaining() < 4) {
        throw new IOException("Incomplete event");
    }
    final ByteOrder bufferOrder = buffer.order();
    buffer.order(ByteOrder.BIG_ENDIAN);
    try {
        int type = buffer.getInt();
        if (type == END_OF_PARTITION_EVENT) {
            return EndOfPartitionEvent.INSTANCE;
        } else if (type == CHECKPOINT_BARRIER_EVENT) {
            long id = buffer.getLong();
            long timestamp = buffer.getLong();
            CheckpointOptions checkpointOptions;
            int checkpointTypeOrdinal = buffer.getInt();
            Preconditions.checkElementIndex(type, CheckpointType.values().length, "Illegal CheckpointType ordinal");
            CheckpointType checkpointType = CheckpointType.values()[checkpointTypeOrdinal];
            if (checkpointType == CheckpointType.FULL_CHECKPOINT) {
                checkpointOptions = CheckpointOptions.forFullCheckpoint();
            } else if (checkpointType == CheckpointType.SAVEPOINT) {
                int len = buffer.getInt();
                byte[] bytes = new byte[len];
                buffer.get(bytes);
                String targetLocation = new String(bytes, STRING_CODING_CHARSET);
                checkpointOptions = CheckpointOptions.forSavepoint(targetLocation);
            } else {
                throw new IOException("Unknown checkpoint type: " + checkpointType);
            }
            return new CheckpointBarrier(id, timestamp, checkpointOptions);
        } else if (type == END_OF_SUPERSTEP_EVENT) {
            return EndOfSuperstepEvent.INSTANCE;
        } else if (type == CANCEL_CHECKPOINT_MARKER_EVENT) {
            long id = buffer.getLong();
            return new CancelCheckpointMarker(id);
        } else if (type == OTHER_EVENT) {
            try {
                final DataInputDeserializer deserializer = new DataInputDeserializer(buffer);
                final String className = deserializer.readUTF();
                final Class<? extends AbstractEvent> clazz;
                try {
                    clazz = classLoader.loadClass(className).asSubclass(AbstractEvent.class);
                } catch (ClassNotFoundException e) {
                    throw new IOException("Could not load event class '" + className + "'.", e);
                } catch (ClassCastException e) {
                    throw new IOException("The class '" + className + "' is not a valid subclass of '" + AbstractEvent.class.getName() + "'.", e);
                }
                final AbstractEvent event = InstantiationUtil.instantiate(clazz, AbstractEvent.class);
                event.read(deserializer);
                return event;
            } catch (Exception e) {
                throw new IOException("Error while deserializing or instantiating event.", e);
            }
        } else {
            throw new IOException("Corrupt byte stream for event");
        }
    } finally {
        buffer.order(bufferOrder);
    }
}

Example 5 with CancelCheckpointMarker

use of org.apache.flink.runtime.io.network.api.CancelCheckpointMarker in project flink by apache.

the class EventSerializerTest method testSerializeDeserializeEvent.

@Test
public void testSerializeDeserializeEvent() throws Exception {
    AbstractEvent[] events = { EndOfPartitionEvent.INSTANCE, EndOfSuperstepEvent.INSTANCE, new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forFullCheckpoint()), new TestTaskEvent(Math.random(), 12361231273L), new CancelCheckpointMarker(287087987329842L) };
    for (AbstractEvent evt : events) {
        ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
        assertTrue(serializedEvent.hasRemaining());
        AbstractEvent deserialized = EventSerializer.fromSerializedEvent(serializedEvent, getClass().getClassLoader());
        assertNotNull(deserialized);
        assertEquals(evt, deserialized);
    }
}