Examples with MasterState org.apache.flink.runtime.checkpoint.MasterState used on opensource projects

Example 1 with MasterState

use of org.apache.flink.runtime.checkpoint.MasterState in project flink by apache.

the class CheckpointMetadataTest method testConstructAndDispose.

@Test
public void testConstructAndDispose() throws Exception {
    final Random rnd = new Random();
    final long checkpointId = rnd.nextInt(Integer.MAX_VALUE) + 1;
    final int numTaskStates = 4;
    final int numSubtaskStates = 16;
    final int numMasterStates = 7;
    Collection<OperatorState> taskStates = CheckpointTestUtils.createOperatorStates(rnd, null, numTaskStates, 0, 0, numSubtaskStates);
    Collection<MasterState> masterStates = CheckpointTestUtils.createRandomMasterStates(rnd, numMasterStates);
    CheckpointMetadata checkpoint = new CheckpointMetadata(checkpointId, taskStates, masterStates);
    assertEquals(checkpointId, checkpoint.getCheckpointId());
    assertEquals(taskStates, checkpoint.getOperatorStates());
    assertEquals(masterStates, checkpoint.getMasterStates());
    assertFalse(checkpoint.getOperatorStates().isEmpty());
    assertFalse(checkpoint.getMasterStates().isEmpty());
    checkpoint.dispose();
    assertTrue(checkpoint.getOperatorStates().isEmpty());
    assertTrue(checkpoint.getMasterStates().isEmpty());
}

Example 2 with MasterState

use of org.apache.flink.runtime.checkpoint.MasterState in project flink by apache.

the class MetadataV3SerializerTest method testCheckpointSerialization.

/**
 * Test checkpoint metadata (de)serialization.
 *
 * @param checkpointId The given checkpointId will write into the metadata.
 * @param operatorStates the given states for all the operators.
 * @param masterStates the masterStates of the given checkpoint/savepoint.
 */
private void testCheckpointSerialization(long checkpointId, Collection<OperatorState> operatorStates, Collection<MasterState> masterStates, @Nullable String basePath) throws IOException {
    MetadataV3Serializer serializer = MetadataV3Serializer.INSTANCE;
    ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos();
    DataOutputStream out = new DataOutputViewStreamWrapper(baos);
    CheckpointMetadata metadata = new CheckpointMetadata(checkpointId, operatorStates, masterStates);
    MetadataV3Serializer.serialize(metadata, out);
    out.close();
    // even if it makes the tests a a tad bit more clumsy
    if (basePath != null) {
        final Path metaPath = new Path(basePath, "_metadata");
        // this is in the temp folder, so it will get automatically deleted
        FileSystem.getLocalFileSystem().create(metaPath, FileSystem.WriteMode.OVERWRITE).close();
    }
    byte[] bytes = baos.toByteArray();
    DataInputStream in = new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(bytes));
    CheckpointMetadata deserialized = serializer.deserialize(in, getClass().getClassLoader(), basePath);
    assertEquals(checkpointId, deserialized.getCheckpointId());
    assertEquals(operatorStates, deserialized.getOperatorStates());
    assertEquals(operatorStates.stream().map(OperatorState::isFullyFinished).collect(Collectors.toList()), deserialized.getOperatorStates().stream().map(OperatorState::isFullyFinished).collect(Collectors.toList()));
    assertEquals(masterStates.size(), deserialized.getMasterStates().size());
    for (Iterator<MasterState> a = masterStates.iterator(), b = deserialized.getMasterStates().iterator(); a.hasNext(); ) {
        CheckpointTestUtils.assertMasterStateEquality(a.next(), b.next());
    }
}

Example 3 with MasterState

use of org.apache.flink.runtime.checkpoint.MasterState in project flink by apache.

the class MetadataV3SerializerTest method testCheckpointWithNoState.

@Test
public void testCheckpointWithNoState() throws Exception {
    final Random rnd = new Random();
    for (int i = 0; i < 100; ++i) {
        final long checkpointId = rnd.nextLong() & 0x7fffffffffffffffL;
        final Collection<OperatorState> taskStates = Collections.emptyList();
        final Collection<MasterState> masterStates = Collections.emptyList();
        testCheckpointSerialization(checkpointId, taskStates, masterStates, null);
    }
}

Example 4 with MasterState

use of org.apache.flink.runtime.checkpoint.MasterState in project flink by apache.

the class MetadataV3SerializerTest method testCheckpointWithOnlyMasterState.

@Test
public void testCheckpointWithOnlyMasterState() throws Exception {
    final Random rnd = new Random();
    final int maxNumMasterStates = 5;
    for (int i = 0; i < 100; ++i) {
        final long checkpointId = rnd.nextLong() & 0x7fffffffffffffffL;
        final Collection<OperatorState> operatorStates = Collections.emptyList();
        final int numMasterStates = rnd.nextInt(maxNumMasterStates) + 1;
        final Collection<MasterState> masterStates = CheckpointTestUtils.createRandomMasterStates(rnd, numMasterStates);
        testCheckpointSerialization(checkpointId, operatorStates, masterStates, null);
    }
}

Example 5 with MasterState

use of org.apache.flink.runtime.checkpoint.MasterState in project flink by apache.

the class MasterHooks method triggerHook.

// ------------------------------------------------------------------------
// checkpoint triggering
// ------------------------------------------------------------------------
/**
 * Trigger master hook and return a completable future with state.
 *
 * @param hook The master hook given
 * @param checkpointId The checkpoint ID of the triggering checkpoint
 * @param timestamp The (informational) timestamp for the triggering checkpoint
 * @param executor An executor that can be used for asynchronous I/O calls
 * @param <T> The type of data produced by the hook
 * @return the completable future with state
 */
public static <T> CompletableFuture<MasterState> triggerHook(MasterTriggerRestoreHook<T> hook, long checkpointId, long timestamp, Executor executor) {
    final String id = hook.getIdentifier();
    final SimpleVersionedSerializer<T> serializer = hook.createCheckpointDataSerializer();
    try {
        // call the hook!
        final CompletableFuture<T> resultFuture = hook.triggerCheckpoint(checkpointId, timestamp, executor);
        if (resultFuture == null) {
            return CompletableFuture.completedFuture(null);
        }
        return resultFuture.thenApply(result -> {
            // if the result of the future is not null, return it as state
            if (result == null) {
                return null;
            } else if (serializer != null) {
                try {
                    final int version = serializer.getVersion();
                    final byte[] bytes = serializer.serialize(result);
                    return new MasterState(id, bytes, version);
                } catch (Throwable t) {
                    ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
                    throw new CompletionException(new FlinkException("Failed to serialize state of master hook '" + id + '\'', t));
                }
            } else {
                throw new CompletionException(new FlinkException("Checkpoint hook '" + id + " is stateful but creates no serializer"));
            }
        }).exceptionally((throwable) -> {
            throw new CompletionException(new FlinkException("Checkpoint master hook '" + id + "' produced an exception", throwable.getCause()));
        });
    } catch (Throwable t) {
        return FutureUtils.completedExceptionally(new FlinkException("Error while triggering checkpoint master hook '" + id + '\'', t));
    }
}