Examples with EventAnnouncement org.apache.flink.runtime.io.network.api.EventAnnouncement used on opensource projects

Example 1 with EventAnnouncement

use of org.apache.flink.runtime.io.network.api.EventAnnouncement in project flink by apache.

the class CheckpointedInputGate method handleEvent.

private Optional<BufferOrEvent> handleEvent(BufferOrEvent bufferOrEvent) throws IOException {
    Class<? extends AbstractEvent> eventClass = bufferOrEvent.getEvent().getClass();
    if (eventClass == CheckpointBarrier.class) {
        CheckpointBarrier checkpointBarrier = (CheckpointBarrier) bufferOrEvent.getEvent();
        barrierHandler.processBarrier(checkpointBarrier, bufferOrEvent.getChannelInfo(), false);
    } else if (eventClass == CancelCheckpointMarker.class) {
        barrierHandler.processCancellationBarrier((CancelCheckpointMarker) bufferOrEvent.getEvent(), bufferOrEvent.getChannelInfo());
    } else if (eventClass == EndOfData.class) {
        inputGate.acknowledgeAllRecordsProcessed(bufferOrEvent.getChannelInfo());
    } else if (eventClass == EndOfPartitionEvent.class) {
        barrierHandler.processEndOfPartition(bufferOrEvent.getChannelInfo());
    } else if (eventClass == EventAnnouncement.class) {
        EventAnnouncement eventAnnouncement = (EventAnnouncement) bufferOrEvent.getEvent();
        AbstractEvent announcedEvent = eventAnnouncement.getAnnouncedEvent();
        checkState(announcedEvent instanceof CheckpointBarrier, "Only CheckpointBarrier announcement are currently supported, but found [%s]", announcedEvent);
        CheckpointBarrier announcedBarrier = (CheckpointBarrier) announcedEvent;
        barrierHandler.processBarrierAnnouncement(announcedBarrier, eventAnnouncement.getSequenceNumber(), bufferOrEvent.getChannelInfo());
    } else if (bufferOrEvent.getEvent().getClass() == EndOfChannelStateEvent.class) {
        upstreamRecoveryTracker.handleEndOfRecovery(bufferOrEvent.getChannelInfo());
    }
    return Optional.of(bufferOrEvent);
}

Example 2 with EventAnnouncement

use of org.apache.flink.runtime.io.network.api.EventAnnouncement in project flink by apache.

the class AlternatingCheckpointsTest method testAllChannelsUnblockedAfterAlignmentTimeout.

@Test
public void testAllChannelsUnblockedAfterAlignmentTimeout() throws Exception {
    int numberOfChannels = 2;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numberOfChannels, getTestBarrierHandlerFactory(target)).withTestChannels().withSyncExecutor().build();
    long alignmentTimeout = 100;
    CheckpointBarrier checkpointBarrier = new CheckpointBarrier(1, clock.relativeTimeMillis(), alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), alignmentTimeout));
    Buffer checkpointBarrierBuffer = toBuffer(checkpointBarrier, false);
    // we set timer on announcement and test channels do not produce announcements by themselves
    send(EventSerializer.toBuffer(new EventAnnouncement(checkpointBarrier, 0), true), 0, gate);
    // emulate blocking channels on aligned barriers
    ((TestInputChannel) gate.getChannel(0)).setBlocked(true);
    send(checkpointBarrierBuffer, 0, gate);
    clock.advanceTime(alignmentTimeout + 1, TimeUnit.MILLISECONDS);
    send(EventSerializer.toBuffer(new EventAnnouncement(checkpointBarrier, 0), true), 1, gate);
    // emulate blocking channels on aligned barriers
    ((TestInputChannel) gate.getChannel(1)).setBlocked(true);
    send(checkpointBarrierBuffer, 1, gate);
    assertThat(target.getTriggeredCheckpointOptions().size(), equalTo(1));
    assertThat(target.getTriggeredCheckpointOptions(), contains(unaligned(CheckpointType.CHECKPOINT, getDefault())));
    assertFalse(((TestInputChannel) gate.getChannel(0)).isBlocked());
    assertFalse(((TestInputChannel) gate.getChannel(1)).isBlocked());
}

Example 3 with EventAnnouncement

use of org.apache.flink.runtime.io.network.api.EventAnnouncement in project flink by apache.

the class AlternatingCheckpointsTest method testSwitchToUnalignedByUpstream.

/**
 * If a checkpoint announcement was processed from one channel and then UC-barrier arrives on
 * another channel, this UC barrier should be processed by the UC controller.
 */
@Test
public void testSwitchToUnalignedByUpstream() throws Exception {
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    try (CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(2, getTestBarrierHandlerFactory(target)).build()) {
        CheckpointBarrier aligned = new CheckpointBarrier(1, clock.relativeTimeMillis(), alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), Integer.MAX_VALUE));
        send(toBuffer(new EventAnnouncement(aligned, 0), true), 0, // process announcement but not the barrier
        gate);
        assertEquals(0, target.triggeredCheckpointCounter);
        send(toBuffer(aligned.asUnaligned(), true), 1, // pretend it came from upstream before the first (AC) barrier was picked
        gate);
        // up
        assertEquals(1, target.triggeredCheckpointCounter);
    }
}

Example 4 with EventAnnouncement

use of org.apache.flink.runtime.io.network.api.EventAnnouncement in project flink by apache.

the class EventSerializer method toSerializedEvent.

// ------------------------------------------------------------------------
// Serialization Logic
// ------------------------------------------------------------------------
public static ByteBuffer toSerializedEvent(AbstractEvent event) throws IOException {
    final Class<?> eventClass = event.getClass();
    if (eventClass == EndOfPartitionEvent.class) {
        return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_PARTITION_EVENT });
    } else if (eventClass == CheckpointBarrier.class) {
        return serializeCheckpointBarrier((CheckpointBarrier) event);
    } else if (eventClass == EndOfSuperstepEvent.class) {
        return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_SUPERSTEP_EVENT });
    } else if (eventClass == EndOfChannelStateEvent.class) {
        return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_CHANNEL_STATE_EVENT });
    } else if (eventClass == EndOfData.class) {
        return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_USER_RECORDS_EVENT, (byte) ((EndOfData) event).getStopMode().ordinal() });
    } else if (eventClass == CancelCheckpointMarker.class) {
        CancelCheckpointMarker marker = (CancelCheckpointMarker) event;
        ByteBuffer buf = ByteBuffer.allocate(12);
        buf.putInt(0, CANCEL_CHECKPOINT_MARKER_EVENT);
        buf.putLong(4, marker.getCheckpointId());
        return buf;
    } else if (eventClass == EventAnnouncement.class) {
        EventAnnouncement announcement = (EventAnnouncement) event;
        ByteBuffer serializedAnnouncedEvent = toSerializedEvent(announcement.getAnnouncedEvent());
        ByteBuffer serializedAnnouncement = ByteBuffer.allocate(2 * Integer.BYTES + serializedAnnouncedEvent.capacity());
        serializedAnnouncement.putInt(0, ANNOUNCEMENT_EVENT);
        serializedAnnouncement.putInt(4, announcement.getSequenceNumber());
        serializedAnnouncement.position(8);
        serializedAnnouncement.put(serializedAnnouncedEvent);
        serializedAnnouncement.flip();
        return serializedAnnouncement;
    } else if (eventClass == SubtaskConnectionDescriptor.class) {
        SubtaskConnectionDescriptor selector = (SubtaskConnectionDescriptor) event;
        ByteBuffer buf = ByteBuffer.allocate(12);
        buf.putInt(VIRTUAL_CHANNEL_SELECTOR_EVENT);
        buf.putInt(selector.getInputSubtaskIndex());
        buf.putInt(selector.getOutputSubtaskIndex());
        buf.flip();
        return buf;
    } else {
        try {
            final DataOutputSerializer serializer = new DataOutputSerializer(128);
            serializer.writeInt(OTHER_EVENT);
            serializer.writeUTF(event.getClass().getName());
            event.write(serializer);
            return serializer.wrapAsByteBuffer();
        } catch (IOException e) {
            throw new IOException("Error while serializing event.", e);
        }
    }
}

Example 5 with EventAnnouncement

use of org.apache.flink.runtime.io.network.api.EventAnnouncement in project flink by apache.

the class EventSerializer method fromSerializedEvent.

public static AbstractEvent fromSerializedEvent(ByteBuffer buffer, ClassLoader classLoader) throws IOException {
    if (buffer.remaining() < 4) {
        throw new IOException("Incomplete event");
    }
    final ByteOrder bufferOrder = buffer.order();
    buffer.order(ByteOrder.BIG_ENDIAN);
    try {
        final int type = buffer.getInt();
        if (type == END_OF_PARTITION_EVENT) {
            return EndOfPartitionEvent.INSTANCE;
        } else if (type == CHECKPOINT_BARRIER_EVENT) {
            return deserializeCheckpointBarrier(buffer);
        } else if (type == END_OF_SUPERSTEP_EVENT) {
            return EndOfSuperstepEvent.INSTANCE;
        } else if (type == END_OF_CHANNEL_STATE_EVENT) {
            return EndOfChannelStateEvent.INSTANCE;
        } else if (type == END_OF_USER_RECORDS_EVENT) {
            return new EndOfData(StopMode.values()[buffer.get()]);
        } else if (type == CANCEL_CHECKPOINT_MARKER_EVENT) {
            long id = buffer.getLong();
            return new CancelCheckpointMarker(id);
        } else if (type == ANNOUNCEMENT_EVENT) {
            int sequenceNumber = buffer.getInt();
            AbstractEvent announcedEvent = fromSerializedEvent(buffer, classLoader);
            return new EventAnnouncement(announcedEvent, sequenceNumber);
        } else if (type == VIRTUAL_CHANNEL_SELECTOR_EVENT) {
            return new SubtaskConnectionDescriptor(buffer.getInt(), buffer.getInt());
        } else if (type == OTHER_EVENT) {
            try {
                final DataInputDeserializer deserializer = new DataInputDeserializer(buffer);
                final String className = deserializer.readUTF();
                final Class<? extends AbstractEvent> clazz;
                try {
                    clazz = classLoader.loadClass(className).asSubclass(AbstractEvent.class);
                } catch (ClassNotFoundException e) {
                    throw new IOException("Could not load event class '" + className + "'.", e);
                } catch (ClassCastException e) {
                    throw new IOException("The class '" + className + "' is not a valid subclass of '" + AbstractEvent.class.getName() + "'.", e);
                }
                final AbstractEvent event = InstantiationUtil.instantiate(clazz, AbstractEvent.class);
                event.read(deserializer);
                return event;
            } catch (Exception e) {
                throw new IOException("Error while deserializing or instantiating event.", e);
            }
        } else {
            throw new IOException("Corrupt byte stream for event");
        }
    } finally {
        buffer.order(bufferOrder);
    }
}