Examples with CheckpointOptions org.apache.flink.runtime.checkpoint.CheckpointOptions used on opensource projects

Example 1 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class HeapSnapshotStrategy method asyncSnapshot.

@Override
public SnapshotResultSupplier<KeyedStateHandle> asyncSnapshot(HeapSnapshotResources<K> syncPartResource, long checkpointId, long timestamp, @Nonnull CheckpointStreamFactory streamFactory, @Nonnull CheckpointOptions checkpointOptions) {
    List<StateMetaInfoSnapshot> metaInfoSnapshots = syncPartResource.getMetaInfoSnapshots();
    if (metaInfoSnapshots.isEmpty()) {
        return snapshotCloseableRegistry -> SnapshotResult.empty();
    }
    final KeyedBackendSerializationProxy<K> serializationProxy = new KeyedBackendSerializationProxy<>(// get a serialized form already at state registration time in the future
    syncPartResource.getKeySerializer(), metaInfoSnapshots, !Objects.equals(UncompressedStreamCompressionDecorator.INSTANCE, keyGroupCompressionDecorator));
    final SupplierWithException<CheckpointStreamWithResultProvider, Exception> checkpointStreamSupplier = localRecoveryConfig.isLocalRecoveryEnabled() && !checkpointOptions.getCheckpointType().isSavepoint() ? () -> createDuplicatingStream(checkpointId, CheckpointedStateScope.EXCLUSIVE, streamFactory, localRecoveryConfig.getLocalStateDirectoryProvider().orElseThrow(LocalRecoveryConfig.localRecoveryNotEnabled())) : () -> createSimpleStream(CheckpointedStateScope.EXCLUSIVE, streamFactory);
    return (snapshotCloseableRegistry) -> {
        final Map<StateUID, Integer> stateNamesToId = syncPartResource.getStateNamesToId();
        final Map<StateUID, StateSnapshot> cowStateStableSnapshots = syncPartResource.getCowStateStableSnapshots();
        final CheckpointStreamWithResultProvider streamWithResultProvider = checkpointStreamSupplier.get();
        snapshotCloseableRegistry.registerCloseable(streamWithResultProvider);
        final CheckpointStateOutputStream localStream = streamWithResultProvider.getCheckpointOutputStream();
        final DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(localStream);
        serializationProxy.write(outView);
        final long[] keyGroupRangeOffsets = new long[keyGroupRange.getNumberOfKeyGroups()];
        for (int keyGroupPos = 0; keyGroupPos < keyGroupRange.getNumberOfKeyGroups(); ++keyGroupPos) {
            int keyGroupId = keyGroupRange.getKeyGroupId(keyGroupPos);
            keyGroupRangeOffsets[keyGroupPos] = localStream.getPos();
            outView.writeInt(keyGroupId);
            for (Map.Entry<StateUID, StateSnapshot> stateSnapshot : cowStateStableSnapshots.entrySet()) {
                StateSnapshot.StateKeyGroupWriter partitionedSnapshot = stateSnapshot.getValue().getKeyGroupWriter();
                try (OutputStream kgCompressionOut = keyGroupCompressionDecorator.decorateWithCompression(localStream)) {
                    DataOutputViewStreamWrapper kgCompressionView = new DataOutputViewStreamWrapper(kgCompressionOut);
                    kgCompressionView.writeShort(stateNamesToId.get(stateSnapshot.getKey()));
                    partitionedSnapshot.writeStateInKeyGroup(kgCompressionView, keyGroupId);
                }
            // this will just close the outer compression stream
            }
        }
        if (snapshotCloseableRegistry.unregisterCloseable(streamWithResultProvider)) {
            KeyGroupRangeOffsets kgOffs = new KeyGroupRangeOffsets(keyGroupRange, keyGroupRangeOffsets);
            SnapshotResult<StreamStateHandle> result = streamWithResultProvider.closeAndFinalizeCheckpointStreamResult();
            return toKeyedStateHandleSnapshotResult(result, kgOffs, KeyGroupsStateHandle::new);
        } else {
            throw new IOException("Stream already unregistered.");
        }
    };
}

Example 2 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class EventSerializer method deserializeCheckpointBarrier.

private static CheckpointBarrier deserializeCheckpointBarrier(ByteBuffer buffer) throws IOException {
    final long id = buffer.getLong();
    final long timestamp = buffer.getLong();
    final byte checkpointTypeCode = buffer.get();
    final SnapshotType snapshotType;
    if (checkpointTypeCode == CHECKPOINT_TYPE_CHECKPOINT) {
        snapshotType = CheckpointType.CHECKPOINT;
    } else if (checkpointTypeCode == CHECKPOINT_TYPE_FULL_CHECKPOINT) {
        snapshotType = CheckpointType.FULL_CHECKPOINT;
    } else if (checkpointTypeCode == CHECKPOINT_TYPE_SAVEPOINT || checkpointTypeCode == CHECKPOINT_TYPE_SAVEPOINT_SUSPEND || checkpointTypeCode == CHECKPOINT_TYPE_SAVEPOINT_TERMINATE) {
        snapshotType = decodeSavepointType(checkpointTypeCode, buffer);
    } else {
        throw new IOException("Unknown checkpoint type code: " + checkpointTypeCode);
    }
    final CheckpointStorageLocationReference locationRef;
    final int locationRefLen = buffer.getInt();
    if (locationRefLen == -1) {
        locationRef = CheckpointStorageLocationReference.getDefault();
    } else {
        byte[] bytes = new byte[locationRefLen];
        buffer.get(bytes);
        locationRef = new CheckpointStorageLocationReference(bytes);
    }
    final CheckpointOptions.AlignmentType alignmentType = CheckpointOptions.AlignmentType.values()[buffer.get()];
    final long alignmentTimeout = buffer.getLong();
    return new CheckpointBarrier(id, timestamp, new CheckpointOptions(snapshotType, locationRef, alignmentType, alignmentTimeout));
}

Example 3 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class EventSerializer method serializeCheckpointBarrier.

private static ByteBuffer serializeCheckpointBarrier(CheckpointBarrier barrier) throws IOException {
    final CheckpointOptions checkpointOptions = barrier.getCheckpointOptions();
    final byte[] locationBytes = checkpointOptions.getTargetLocation().isDefaultReference() ? null : checkpointOptions.getTargetLocation().getReferenceBytes();
    final ByteBuffer buf = ByteBuffer.allocate(38 + (locationBytes == null ? 0 : locationBytes.length));
    buf.putInt(CHECKPOINT_BARRIER_EVENT);
    buf.putLong(barrier.getId());
    buf.putLong(barrier.getTimestamp());
    // we do not use checkpointType.ordinal() here to make the serialization robust
    // against changes in the enum (such as changes in the order of the values)
    final SnapshotType snapshotType = checkpointOptions.getCheckpointType();
    if (snapshotType.isSavepoint()) {
        encodeSavepointType(snapshotType, buf);
    } else if (snapshotType.equals(CheckpointType.CHECKPOINT)) {
        buf.put(CHECKPOINT_TYPE_CHECKPOINT);
    } else if (snapshotType.equals(CheckpointType.FULL_CHECKPOINT)) {
        buf.put(CHECKPOINT_TYPE_FULL_CHECKPOINT);
    } else {
        throw new IOException("Unknown checkpoint type: " + snapshotType);
    }
    if (locationBytes == null) {
        buf.putInt(-1);
    } else {
        buf.putInt(locationBytes.length);
        buf.put(locationBytes);
    }
    buf.put((byte) checkpointOptions.getAlignment().ordinal());
    buf.putLong(checkpointOptions.getAlignedCheckpointTimeout());
    buf.flip();
    return buf;
}

Example 4 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class DefaultOperatorStateBackendSnapshotStrategy method asyncSnapshot.

@Override
public SnapshotResultSupplier<OperatorStateHandle> asyncSnapshot(DefaultOperatorStateBackendSnapshotResources syncPartResource, long checkpointId, long timestamp, @Nonnull CheckpointStreamFactory streamFactory, @Nonnull CheckpointOptions checkpointOptions) {
    Map<String, PartitionableListState<?>> registeredOperatorStatesDeepCopies = syncPartResource.getRegisteredOperatorStatesDeepCopies();
    Map<String, BackendWritableBroadcastState<?, ?>> registeredBroadcastStatesDeepCopies = syncPartResource.getRegisteredBroadcastStatesDeepCopies();
    if (registeredBroadcastStatesDeepCopies.isEmpty() && registeredOperatorStatesDeepCopies.isEmpty()) {
        return snapshotCloseableRegistry -> SnapshotResult.empty();
    }
    return (snapshotCloseableRegistry) -> {
        CheckpointStateOutputStream localOut = streamFactory.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE);
        snapshotCloseableRegistry.registerCloseable(localOut);
        // get the registered operator state infos ...
        List<StateMetaInfoSnapshot> operatorMetaInfoSnapshots = new ArrayList<>(registeredOperatorStatesDeepCopies.size());
        for (Map.Entry<String, PartitionableListState<?>> entry : registeredOperatorStatesDeepCopies.entrySet()) {
            operatorMetaInfoSnapshots.add(entry.getValue().getStateMetaInfo().snapshot());
        }
        // ... get the registered broadcast operator state infos ...
        List<StateMetaInfoSnapshot> broadcastMetaInfoSnapshots = new ArrayList<>(registeredBroadcastStatesDeepCopies.size());
        for (Map.Entry<String, BackendWritableBroadcastState<?, ?>> entry : registeredBroadcastStatesDeepCopies.entrySet()) {
            broadcastMetaInfoSnapshots.add(entry.getValue().getStateMetaInfo().snapshot());
        }
        // ... write them all in the checkpoint stream ...
        DataOutputView dov = new DataOutputViewStreamWrapper(localOut);
        OperatorBackendSerializationProxy backendSerializationProxy = new OperatorBackendSerializationProxy(operatorMetaInfoSnapshots, broadcastMetaInfoSnapshots);
        backendSerializationProxy.write(dov);
        // ... and then go for the states ...
        // we put BOTH normal and broadcast state metadata here
        int initialMapCapacity = registeredOperatorStatesDeepCopies.size() + registeredBroadcastStatesDeepCopies.size();
        final Map<String, OperatorStateHandle.StateMetaInfo> writtenStatesMetaData = new HashMap<>(initialMapCapacity);
        for (Map.Entry<String, PartitionableListState<?>> entry : registeredOperatorStatesDeepCopies.entrySet()) {
            PartitionableListState<?> value = entry.getValue();
            long[] partitionOffsets = value.write(localOut);
            OperatorStateHandle.Mode mode = value.getStateMetaInfo().getAssignmentMode();
            writtenStatesMetaData.put(entry.getKey(), new OperatorStateHandle.StateMetaInfo(partitionOffsets, mode));
        }
        // ... and the broadcast states themselves ...
        for (Map.Entry<String, BackendWritableBroadcastState<?, ?>> entry : registeredBroadcastStatesDeepCopies.entrySet()) {
            BackendWritableBroadcastState<?, ?> value = entry.getValue();
            long[] partitionOffsets = { value.write(localOut) };
            OperatorStateHandle.Mode mode = value.getStateMetaInfo().getAssignmentMode();
            writtenStatesMetaData.put(entry.getKey(), new OperatorStateHandle.StateMetaInfo(partitionOffsets, mode));
        }
        // ... and, finally, create the state handle.
        OperatorStateHandle retValue = null;
        if (snapshotCloseableRegistry.unregisterCloseable(localOut)) {
            StreamStateHandle stateHandle = localOut.closeAndGetHandle();
            if (stateHandle != null) {
                retValue = new OperatorStreamStateHandle(writtenStatesMetaData, stateHandle);
            }
            return SnapshotResult.of(retValue);
        } else {
            throw new IOException("Stream was already unregistered.");
        }
    };
}

Example 5 with CheckpointOptions

use of org.apache.flink.runtime.checkpoint.CheckpointOptions in project flink by apache.

the class SavepointStateBackendSwitchTestBase method takeSavepoint.

private void takeSavepoint(CheckpointableKeyedStateBackend<String> keyedBackend, File pathToWrite, MapStateDescriptor<Long, Long> stateDescr, ValueStateDescriptor<Long> valueStateDescriptor, ListStateDescriptor<Long> listStateDescriptor, Integer namespace1, Integer namespace2, Integer namespace3, Integer namespace4) throws Exception {
    InternalMapState<String, Integer, Long, Long> mapState = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);
    InternalValueState<String, Integer, Long> valueState = keyedBackend.createInternalState(IntSerializer.INSTANCE, valueStateDescriptor);
    InternalListState<String, Integer, Long> listState = keyedBackend.createInternalState(IntSerializer.INSTANCE, listStateDescriptor);
    keyedBackend.setCurrentKey("abc");
    mapState.setCurrentNamespace(namespace1);
    mapState.put(33L, 33L);
    mapState.put(55L, 55L);
    mapState.setCurrentNamespace(namespace2);
    mapState.put(22L, 22L);
    mapState.put(11L, 11L);
    listState.setCurrentNamespace(namespace2);
    listState.add(4L);
    listState.add(5L);
    listState.add(6L);
    mapState.setCurrentNamespace(namespace3);
    mapState.put(44L, 44L);
    keyedBackend.setCurrentKey("mno");
    mapState.setCurrentNamespace(namespace3);
    mapState.put(11L, 11L);
    mapState.put(22L, 22L);
    mapState.put(33L, 33L);
    mapState.put(44L, 44L);
    mapState.put(55L, 55L);
    valueState.setCurrentNamespace(namespace3);
    valueState.update(1239L);
    listState.setCurrentNamespace(namespace3);
    listState.add(1L);
    listState.add(2L);
    listState.add(3L);
    mapState.setCurrentNamespace(namespace4);
    mapState.put(1L, 1L);
    // HEAP state backend will keep an empty map as an entry in the underlying State Table
    // we should skip such entries when serializing
    Iterator<Map.Entry<Long, Long>> iterator = mapState.iterator();
    while (iterator.hasNext()) {
        iterator.next();
        iterator.remove();
    }
    KeyGroupedInternalPriorityQueue<TimerHeapInternalTimer<String, Integer>> priorityQueue = keyedBackend.create("event-time", new TimerSerializer<>(keyedBackend.getKeySerializer(), IntSerializer.INSTANCE));
    priorityQueue.add(new TimerHeapInternalTimer<>(1234L, "mno", namespace3));
    priorityQueue.add(new TimerHeapInternalTimer<>(2345L, "mno", namespace2));
    priorityQueue.add(new TimerHeapInternalTimer<>(3456L, "mno", namespace3));
    SnapshotStrategyRunner<KeyedStateHandle, ? extends FullSnapshotResources<?>> savepointRunner = StreamOperatorStateHandler.prepareCanonicalSavepoint(keyedBackend, new CloseableRegistry());
    RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = savepointRunner.snapshot(0L, 0L, new MemCheckpointStreamFactory(4 * 1024 * 1024), new CheckpointOptions(SavepointType.savepoint(SavepointFormatType.CANONICAL), CheckpointStorageLocationReference.getDefault()));
    snapshot.run();
    try (BufferedOutputStream bis = new BufferedOutputStream(new FileOutputStream(pathToWrite))) {
        InstantiationUtil.serializeObject(bis, snapshot.get());
    }
}