Examples with CheckpointBarrier org.apache.flink.runtime.io.network.api.CheckpointBarrier used on opensource projects

Example 1 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class PipelinedSubpartition method processPriorityBuffer.

private boolean processPriorityBuffer(BufferConsumer bufferConsumer, int partialRecordLength) {
    buffers.addPriorityElement(new BufferConsumerWithPartialRecordLength(bufferConsumer, partialRecordLength));
    final int numPriorityElements = buffers.getNumPriorityElements();
    CheckpointBarrier barrier = parseCheckpointBarrier(bufferConsumer);
    if (barrier != null) {
        checkState(barrier.getCheckpointOptions().isUnalignedCheckpoint(), "Only unaligned checkpoints should be priority events");
        final Iterator<BufferConsumerWithPartialRecordLength> iterator = buffers.iterator();
        Iterators.advance(iterator, numPriorityElements);
        List<Buffer> inflightBuffers = new ArrayList<>();
        while (iterator.hasNext()) {
            BufferConsumer buffer = iterator.next().getBufferConsumer();
            if (buffer.isBuffer()) {
                try (BufferConsumer bc = buffer.copy()) {
                    inflightBuffers.add(bc.build());
                }
            }
        }
        if (!inflightBuffers.isEmpty()) {
            channelStateWriter.addOutputData(barrier.getId(), subpartitionInfo, ChannelStateWriter.SEQUENCE_NUMBER_UNKNOWN, inflightBuffers.toArray(new Buffer[0]));
        }
    }
    return numPriorityElements == 1 && // if subpartition is blocked then downstream doesn't expect any
    !isBlocked;
// notifications
}

Example 2 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class PipelinedSubpartition method parseCheckpointBarrier.

@Nullable
private CheckpointBarrier parseCheckpointBarrier(BufferConsumer bufferConsumer) {
    CheckpointBarrier barrier;
    try (BufferConsumer bc = bufferConsumer.copy()) {
        Buffer buffer = bc.build();
        try {
            final AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
            barrier = event instanceof CheckpointBarrier ? (CheckpointBarrier) event : null;
        } catch (IOException e) {
            throw new IllegalStateException("Should always be able to deserialize in-memory event", e);
        } finally {
            buffer.recycleBuffer();
        }
    }
    return barrier;
}

Example 3 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class EventSerializer method deserializeCheckpointBarrier.

private static CheckpointBarrier deserializeCheckpointBarrier(ByteBuffer buffer) throws IOException {
    final long id = buffer.getLong();
    final long timestamp = buffer.getLong();
    final byte checkpointTypeCode = buffer.get();
    final SnapshotType snapshotType;
    if (checkpointTypeCode == CHECKPOINT_TYPE_CHECKPOINT) {
        snapshotType = CheckpointType.CHECKPOINT;
    } else if (checkpointTypeCode == CHECKPOINT_TYPE_FULL_CHECKPOINT) {
        snapshotType = CheckpointType.FULL_CHECKPOINT;
    } else if (checkpointTypeCode == CHECKPOINT_TYPE_SAVEPOINT || checkpointTypeCode == CHECKPOINT_TYPE_SAVEPOINT_SUSPEND || checkpointTypeCode == CHECKPOINT_TYPE_SAVEPOINT_TERMINATE) {
        snapshotType = decodeSavepointType(checkpointTypeCode, buffer);
    } else {
        throw new IOException("Unknown checkpoint type code: " + checkpointTypeCode);
    }
    final CheckpointStorageLocationReference locationRef;
    final int locationRefLen = buffer.getInt();
    if (locationRefLen == -1) {
        locationRef = CheckpointStorageLocationReference.getDefault();
    } else {
        byte[] bytes = new byte[locationRefLen];
        buffer.get(bytes);
        locationRef = new CheckpointStorageLocationReference(bytes);
    }
    final CheckpointOptions.AlignmentType alignmentType = CheckpointOptions.AlignmentType.values()[buffer.get()];
    final long alignmentTimeout = buffer.getLong();
    return new CheckpointBarrier(id, timestamp, new CheckpointOptions(snapshotType, locationRef, alignmentType, alignmentTimeout));
}

Example 4 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class AlternatingCollectingBarriers method alignmentTimeout.

@Override
public BarrierHandlerState alignmentTimeout(Controller controller, CheckpointBarrier checkpointBarrier) throws IOException, CheckpointException {
    state.prioritizeAllAnnouncements();
    CheckpointBarrier unalignedBarrier = checkpointBarrier.asUnaligned();
    controller.initInputsCheckpoint(unalignedBarrier);
    for (CheckpointableInput input : state.getInputs()) {
        input.checkpointStarted(unalignedBarrier);
    }
    controller.triggerGlobalCheckpoint(unalignedBarrier);
    return new AlternatingCollectingBarriersUnaligned(true, state);
}

Example 5 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class CheckpointedInputGate method handleEvent.

private Optional<BufferOrEvent> handleEvent(BufferOrEvent bufferOrEvent) throws IOException {
    Class<? extends AbstractEvent> eventClass = bufferOrEvent.getEvent().getClass();
    if (eventClass == CheckpointBarrier.class) {
        CheckpointBarrier checkpointBarrier = (CheckpointBarrier) bufferOrEvent.getEvent();
        barrierHandler.processBarrier(checkpointBarrier, bufferOrEvent.getChannelInfo(), false);
    } else if (eventClass == CancelCheckpointMarker.class) {
        barrierHandler.processCancellationBarrier((CancelCheckpointMarker) bufferOrEvent.getEvent(), bufferOrEvent.getChannelInfo());
    } else if (eventClass == EndOfData.class) {
        inputGate.acknowledgeAllRecordsProcessed(bufferOrEvent.getChannelInfo());
    } else if (eventClass == EndOfPartitionEvent.class) {
        barrierHandler.processEndOfPartition(bufferOrEvent.getChannelInfo());
    } else if (eventClass == EventAnnouncement.class) {
        EventAnnouncement eventAnnouncement = (EventAnnouncement) bufferOrEvent.getEvent();
        AbstractEvent announcedEvent = eventAnnouncement.getAnnouncedEvent();
        checkState(announcedEvent instanceof CheckpointBarrier, "Only CheckpointBarrier announcement are currently supported, but found [%s]", announcedEvent);
        CheckpointBarrier announcedBarrier = (CheckpointBarrier) announcedEvent;
        barrierHandler.processBarrierAnnouncement(announcedBarrier, eventAnnouncement.getSequenceNumber(), bufferOrEvent.getChannelInfo());
    } else if (bufferOrEvent.getEvent().getClass() == EndOfChannelStateEvent.class) {
        upstreamRecoveryTracker.handleEndOfRecovery(bufferOrEvent.getChannelInfo());
    }
    return Optional.of(bufferOrEvent);
}