Examples with OperatorState org.apache.flink.runtime.checkpoint.OperatorState used on opensource projects

Example 1 with OperatorState

use of org.apache.flink.runtime.checkpoint.OperatorState in project flink by apache.

the class CheckpointMetadataTest method testConstructAndDispose.

@Test
public void testConstructAndDispose() throws Exception {
    final Random rnd = new Random();
    final long checkpointId = rnd.nextInt(Integer.MAX_VALUE) + 1;
    final int numTaskStates = 4;
    final int numSubtaskStates = 16;
    final int numMasterStates = 7;
    Collection<OperatorState> taskStates = CheckpointTestUtils.createOperatorStates(rnd, null, numTaskStates, 0, 0, numSubtaskStates);
    Collection<MasterState> masterStates = CheckpointTestUtils.createRandomMasterStates(rnd, numMasterStates);
    CheckpointMetadata checkpoint = new CheckpointMetadata(checkpointId, taskStates, masterStates);
    assertEquals(checkpointId, checkpoint.getCheckpointId());
    assertEquals(taskStates, checkpoint.getOperatorStates());
    assertEquals(masterStates, checkpoint.getMasterStates());
    assertFalse(checkpoint.getOperatorStates().isEmpty());
    assertFalse(checkpoint.getMasterStates().isEmpty());
    checkpoint.dispose();
    assertTrue(checkpoint.getOperatorStates().isEmpty());
    assertTrue(checkpoint.getMasterStates().isEmpty());
}

Example 2 with OperatorState

use of org.apache.flink.runtime.checkpoint.OperatorState in project flink by apache.

the class MetadataV3SerializerTest method testCheckpointSerialization.

/**
 * Test checkpoint metadata (de)serialization.
 *
 * @param checkpointId The given checkpointId will write into the metadata.
 * @param operatorStates the given states for all the operators.
 * @param masterStates the masterStates of the given checkpoint/savepoint.
 */
private void testCheckpointSerialization(long checkpointId, Collection<OperatorState> operatorStates, Collection<MasterState> masterStates, @Nullable String basePath) throws IOException {
    MetadataV3Serializer serializer = MetadataV3Serializer.INSTANCE;
    ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos();
    DataOutputStream out = new DataOutputViewStreamWrapper(baos);
    CheckpointMetadata metadata = new CheckpointMetadata(checkpointId, operatorStates, masterStates);
    MetadataV3Serializer.serialize(metadata, out);
    out.close();
    // even if it makes the tests a a tad bit more clumsy
    if (basePath != null) {
        final Path metaPath = new Path(basePath, "_metadata");
        // this is in the temp folder, so it will get automatically deleted
        FileSystem.getLocalFileSystem().create(metaPath, FileSystem.WriteMode.OVERWRITE).close();
    }
    byte[] bytes = baos.toByteArray();
    DataInputStream in = new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(bytes));
    CheckpointMetadata deserialized = serializer.deserialize(in, getClass().getClassLoader(), basePath);
    assertEquals(checkpointId, deserialized.getCheckpointId());
    assertEquals(operatorStates, deserialized.getOperatorStates());
    assertEquals(operatorStates.stream().map(OperatorState::isFullyFinished).collect(Collectors.toList()), deserialized.getOperatorStates().stream().map(OperatorState::isFullyFinished).collect(Collectors.toList()));
    assertEquals(masterStates.size(), deserialized.getMasterStates().size());
    for (Iterator<MasterState> a = masterStates.iterator(), b = deserialized.getMasterStates().iterator(); a.hasNext(); ) {
        CheckpointTestUtils.assertMasterStateEquality(a.next(), b.next());
    }
}

Example 3 with OperatorState

use of org.apache.flink.runtime.checkpoint.OperatorState in project flink by apache.

the class MetadataV3SerializerTest method testCheckpointWithNoState.

@Test
public void testCheckpointWithNoState() throws Exception {
    final Random rnd = new Random();
    for (int i = 0; i < 100; ++i) {
        final long checkpointId = rnd.nextLong() & 0x7fffffffffffffffL;
        final Collection<OperatorState> taskStates = Collections.emptyList();
        final Collection<MasterState> masterStates = Collections.emptyList();
        testCheckpointSerialization(checkpointId, taskStates, masterStates, null);
    }
}

Example 4 with OperatorState

use of org.apache.flink.runtime.checkpoint.OperatorState in project flink by apache.

the class MetadataV3SerializerTest method testCheckpointWithOnlyMasterState.

@Test
public void testCheckpointWithOnlyMasterState() throws Exception {
    final Random rnd = new Random();
    final int maxNumMasterStates = 5;
    for (int i = 0; i < 100; ++i) {
        final long checkpointId = rnd.nextLong() & 0x7fffffffffffffffL;
        final Collection<OperatorState> operatorStates = Collections.emptyList();
        final int numMasterStates = rnd.nextInt(maxNumMasterStates) + 1;
        final Collection<MasterState> masterStates = CheckpointTestUtils.createRandomMasterStates(rnd, numMasterStates);
        testCheckpointSerialization(checkpointId, operatorStates, masterStates, null);
    }
}

Example 5 with OperatorState

use of org.apache.flink.runtime.checkpoint.OperatorState in project flink by apache.

the class SavepointWriter method write.

/**
 * Write out a new or updated savepoint.
 *
 * @param path The path to where the savepoint should be written.
 */
public final void write(String path) {
    final Path savepointPath = new Path(path);
    List<StateBootstrapTransformationWithID<?>> newOperatorTransformations = metadata.getNewOperators();
    DataStream<OperatorState> newOperatorStates = writeOperatorStates(newOperatorTransformations, configuration, savepointPath);
    List<OperatorState> existingOperators = metadata.getExistingOperators();
    DataStream<OperatorState> finalOperatorStates;
    if (existingOperators.isEmpty()) {
        finalOperatorStates = newOperatorStates;
    } else {
        DataStream<OperatorState> existingOperatorStates = newOperatorStates.getExecutionEnvironment().fromCollection(existingOperators).name("existingOperatorStates");
        existingOperatorStates.flatMap(new StatePathExtractor()).setParallelism(1).addSink(new OutputFormatSinkFunction<>(new FileCopyFunction(path)));
        finalOperatorStates = newOperatorStates.union(existingOperatorStates);
    }
    finalOperatorStates.transform("reduce(OperatorState)", TypeInformation.of(CheckpointMetadata.class), new GroupReduceOperator<>(new MergeOperatorStates(metadata.getMasterStates()))).forceNonParallel().addSink(new OutputFormatSinkFunction<>(new SavepointOutputFormat(savepointPath))).setParallelism(1).name(path);
}