Examples with SnapshotType org.apache.flink.runtime.checkpoint.SnapshotType used on opensource projects

Example 1 with SnapshotType

use of org.apache.flink.runtime.checkpoint.SnapshotType in project flink by apache.

the class EventSerializer method deserializeCheckpointBarrier.

private static CheckpointBarrier deserializeCheckpointBarrier(ByteBuffer buffer) throws IOException {
    final long id = buffer.getLong();
    final long timestamp = buffer.getLong();
    final byte checkpointTypeCode = buffer.get();
    final SnapshotType snapshotType;
    if (checkpointTypeCode == CHECKPOINT_TYPE_CHECKPOINT) {
        snapshotType = CheckpointType.CHECKPOINT;
    } else if (checkpointTypeCode == CHECKPOINT_TYPE_FULL_CHECKPOINT) {
        snapshotType = CheckpointType.FULL_CHECKPOINT;
    } else if (checkpointTypeCode == CHECKPOINT_TYPE_SAVEPOINT || checkpointTypeCode == CHECKPOINT_TYPE_SAVEPOINT_SUSPEND || checkpointTypeCode == CHECKPOINT_TYPE_SAVEPOINT_TERMINATE) {
        snapshotType = decodeSavepointType(checkpointTypeCode, buffer);
    } else {
        throw new IOException("Unknown checkpoint type code: " + checkpointTypeCode);
    }
    final CheckpointStorageLocationReference locationRef;
    final int locationRefLen = buffer.getInt();
    if (locationRefLen == -1) {
        locationRef = CheckpointStorageLocationReference.getDefault();
    } else {
        byte[] bytes = new byte[locationRefLen];
        buffer.get(bytes);
        locationRef = new CheckpointStorageLocationReference(bytes);
    }
    final CheckpointOptions.AlignmentType alignmentType = CheckpointOptions.AlignmentType.values()[buffer.get()];
    final long alignmentTimeout = buffer.getLong();
    return new CheckpointBarrier(id, timestamp, new CheckpointOptions(snapshotType, locationRef, alignmentType, alignmentTimeout));
}

Example 2 with SnapshotType

use of org.apache.flink.runtime.checkpoint.SnapshotType in project flink by apache.

the class EventSerializer method serializeCheckpointBarrier.

private static ByteBuffer serializeCheckpointBarrier(CheckpointBarrier barrier) throws IOException {
    final CheckpointOptions checkpointOptions = barrier.getCheckpointOptions();
    final byte[] locationBytes = checkpointOptions.getTargetLocation().isDefaultReference() ? null : checkpointOptions.getTargetLocation().getReferenceBytes();
    final ByteBuffer buf = ByteBuffer.allocate(38 + (locationBytes == null ? 0 : locationBytes.length));
    buf.putInt(CHECKPOINT_BARRIER_EVENT);
    buf.putLong(barrier.getId());
    buf.putLong(barrier.getTimestamp());
    // we do not use checkpointType.ordinal() here to make the serialization robust
    // against changes in the enum (such as changes in the order of the values)
    final SnapshotType snapshotType = checkpointOptions.getCheckpointType();
    if (snapshotType.isSavepoint()) {
        encodeSavepointType(snapshotType, buf);
    } else if (snapshotType.equals(CheckpointType.CHECKPOINT)) {
        buf.put(CHECKPOINT_TYPE_CHECKPOINT);
    } else if (snapshotType.equals(CheckpointType.FULL_CHECKPOINT)) {
        buf.put(CHECKPOINT_TYPE_FULL_CHECKPOINT);
    } else {
        throw new IOException("Unknown checkpoint type: " + snapshotType);
    }
    if (locationBytes == null) {
        buf.putInt(-1);
    } else {
        buf.putInt(locationBytes.length);
        buf.put(locationBytes);
    }
    buf.put((byte) checkpointOptions.getAlignment().ordinal());
    buf.putLong(checkpointOptions.getAlignedCheckpointTimeout());
    buf.flip();
    return buf;
}

Example 3 with SnapshotType

use of org.apache.flink.runtime.checkpoint.SnapshotType in project flink by apache.

the class AlternatingCheckpointsTest method testPreviousHandlerReset.

@Test
public void testPreviousHandlerReset() throws Exception {
    SingleInputGate inputGate = new SingleInputGateBuilder().setNumberOfChannels(2).build();
    TestInputChannel[] channels = { new TestInputChannel(inputGate, 0), new TestInputChannel(inputGate, 1) };
    inputGate.setInputChannels(channels);
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    SingleCheckpointBarrierHandler barrierHandler = getTestBarrierHandlerFactory(target).create(inputGate);
    for (int i = 0; i < 4; i++) {
        int channel = i % 2;
        SnapshotType type = channel == 0 ? SavepointType.savepoint(SavepointFormatType.CANONICAL) : CHECKPOINT;
        target.setNextExpectedCheckpointId(-1);
        if (type.isSavepoint()) {
            channels[channel].setBlocked(true);
        }
        barrierHandler.processBarrier(new CheckpointBarrier(i, clock.relativeTimeMillis(), new CheckpointOptions(type, getDefault())), new InputChannelInfo(0, channel), false);
        if (type.isSavepoint()) {
            assertTrue(channels[channel].isBlocked());
            assertFalse(channels[(channel + 1) % 2].isBlocked());
        } else {
            assertFalse(channels[0].isBlocked());
            assertFalse(channels[1].isBlocked());
        }
        assertTrue(barrierHandler.isCheckpointPending());
        assertFalse(barrierHandler.getAllBarriersReceivedFuture(i).isDone());
        channels[0].setBlocked(false);
        channels[1].setBlocked(false);
    }
}

Example 4 with SnapshotType

use of org.apache.flink.runtime.checkpoint.SnapshotType in project flink by apache.

the class AlternatingCheckpointsTest method testAlternation.

@Test
public void testAlternation() throws Exception {
    int numBarriers = 123;
    int numChannels = 123;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    try (CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).build()) {
        List<Long> barriers = new ArrayList<>();
        for (long barrier = 0; barrier < numBarriers; barrier++) {
            barriers.add(barrier);
            SnapshotType type = barrier % 2 == 0 ? CHECKPOINT : SavepointType.savepoint(SavepointFormatType.CANONICAL);
            for (int channel = 0; channel < numChannels; channel++) {
                send(barrier(barrier, clock.relativeTimeMillis(), alignedNoTimeout(type, getDefault())).retainBuffer(), channel, gate);
            }
        }
        assertEquals(barriers, target.triggeredCheckpoints);
    }
}

Example 5 with SnapshotType

use of org.apache.flink.runtime.checkpoint.SnapshotType in project flink by apache.

the class StreamTask method performCheckpoint.

private boolean performCheckpoint(CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions, CheckpointMetricsBuilder checkpointMetrics) throws Exception {
    final SnapshotType checkpointType = checkpointOptions.getCheckpointType();
    LOG.debug("Starting checkpoint {} {} on task {}", checkpointMetaData.getCheckpointId(), checkpointType, getName());
    if (isRunning) {
        actionExecutor.runThrowing(() -> {
            if (isSynchronous(checkpointType)) {
                setSynchronousSavepoint(checkpointMetaData.getCheckpointId());
            }
            if (areCheckpointsWithFinishedTasksEnabled() && endOfDataReceived && this.finalCheckpointMinId == null) {
                this.finalCheckpointMinId = checkpointMetaData.getCheckpointId();
            }
            subtaskCheckpointCoordinator.checkpointState(checkpointMetaData, checkpointOptions, checkpointMetrics, operatorChain, finishedOperators, this::isRunning);
        });
        return true;
    } else {
        actionExecutor.runThrowing(() -> {
            // we cannot perform our checkpoint - let the downstream operators know that
            // they
            // should not wait for any input from this operator
            // we cannot broadcast the cancellation markers on the 'operator chain',
            // because it may not
            // yet be created
            final CancelCheckpointMarker message = new CancelCheckpointMarker(checkpointMetaData.getCheckpointId());
            recordWriter.broadcastEvent(message);
        });
        return false;
    }
}