Search in sources :

Example 11 with SavepointFormatType

use of org.apache.flink.core.execution.SavepointFormatType in project flink by apache.

the class SavepointFormatITCase method testTriggerSavepointAndResumeWithFileBasedCheckpointsAndRelocateBasePath.

@ParameterizedTest(name = "[{index}] {0}, {1}")
@MethodSource("parameters")
public void testTriggerSavepointAndResumeWithFileBasedCheckpointsAndRelocateBasePath(SavepointFormatType formatType, StateBackendConfig stateBackendConfig) throws Exception {
    final int numTaskManagers = 2;
    final int numSlotsPerTaskManager = 2;
    final Configuration config = stateBackendConfig.getConfiguration();
    config.set(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointsDir.toUri().toString());
    final MiniClusterWithClientResource miniClusterResource = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder().setConfiguration(config).setNumberTaskManagers(numTaskManagers).setNumberSlotsPerTaskManager(numSlotsPerTaskManager).build());
    miniClusterResource.before();
    try {
        final String savepointPath = submitJobAndTakeSavepoint(miniClusterResource, formatType, stateBackendConfig.getCheckpointsBeforeSavepoint(), config);
        final CheckpointMetadata metadata = loadCheckpointMetadata(savepointPath);
        final OperatorState operatorState = metadata.getOperatorStates().stream().filter(hasKeyedState()).findFirst().get();
        operatorState.getStates().stream().flatMap(subtaskState -> subtaskState.getManagedKeyedState().stream()).forEach(handle -> validateState(handle, formatType, stateBackendConfig));
        relocateAndVerify(miniClusterResource, savepointPath, renamedSavepointDir, config);
    } finally {
        miniClusterResource.after();
    }
}
Also used : KeyGroupsStateHandle(org.apache.flink.runtime.state.KeyGroupsStateHandle) BiFunction(java.util.function.BiFunction) OperatorState(org.apache.flink.runtime.checkpoint.OperatorState) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) IncrementalRemoteKeyedStateHandle(org.apache.flink.runtime.state.IncrementalRemoteKeyedStateHandle) MemorySize(org.apache.flink.configuration.MemorySize) MiniClusterResourceConfiguration(org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration) BasicTypeInfo(org.apache.flink.api.common.typeinfo.BasicTypeInfo) CoreMatchers.instanceOf(org.hamcrest.CoreMatchers.instanceOf) CommonTestUtils.waitForAllTaskRunning(org.apache.flink.runtime.testutils.CommonTestUtils.waitForAllTaskRunning) Arrays.asList(java.util.Arrays.asList) ChangelogStateBackendHandle(org.apache.flink.runtime.state.changelog.ChangelogStateBackendHandle) Level(org.slf4j.event.Level) Checkpoints(org.apache.flink.runtime.checkpoint.Checkpoints) Path(java.nio.file.Path) MethodSource(org.junit.jupiter.params.provider.MethodSource) LoggerAuditingExtension(org.apache.flink.testutils.logging.LoggerAuditingExtension) Predicate(java.util.function.Predicate) CompletedCheckpointStorageLocation(org.apache.flink.runtime.state.CompletedCheckpointStorageLocation) KeyedStateHandle(org.apache.flink.runtime.state.KeyedStateHandle) Arguments(org.junit.jupiter.params.provider.Arguments) CheckpointingOptions(org.apache.flink.configuration.CheckpointingOptions) CheckpointMetadata(org.apache.flink.runtime.checkpoint.metadata.CheckpointMetadata) List(java.util.List) ValueState(org.apache.flink.api.common.state.ValueState) ClusterClient(org.apache.flink.client.program.ClusterClient) TempDir(org.junit.jupiter.api.io.TempDir) Optional(java.util.Optional) NotNull(org.jetbrains.annotations.NotNull) StreamExecutionEnvironment(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment) DataInputStream(java.io.DataInputStream) SavepointFormatType(org.apache.flink.core.execution.SavepointFormatType) AbstractFsCheckpointStorageAccess(org.apache.flink.runtime.state.filesystem.AbstractFsCheckpointStorageAccess) RestoreMode(org.apache.flink.runtime.jobgraph.RestoreMode) RichMapFunction(org.apache.flink.api.common.functions.RichMapFunction) RegisterExtension(org.junit.jupiter.api.extension.RegisterExtension) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) LinkedList(java.util.LinkedList) MiniClusterWithClientResource(org.apache.flink.test.util.MiniClusterWithClientResource) DiscardingSink(org.apache.flink.streaming.api.functions.sink.DiscardingSink) ValueStateDescriptor(org.apache.flink.api.common.state.ValueStateDescriptor) StateChangelogOptions(org.apache.flink.configuration.StateChangelogOptions) Configuration(org.apache.flink.configuration.Configuration) IOException(java.io.IOException) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) JobID(org.apache.flink.api.common.JobID) RuntimeExecutionMode(org.apache.flink.api.common.RuntimeExecutionMode) SavepointRestoreSettings(org.apache.flink.runtime.jobgraph.SavepointRestoreSettings) SavepointKeyedStateHandle(org.apache.flink.runtime.state.SavepointKeyedStateHandle) StateBackendOptions(org.apache.flink.configuration.StateBackendOptions) MiniClusterResourceConfiguration(org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration) Configuration(org.apache.flink.configuration.Configuration) MiniClusterWithClientResource(org.apache.flink.test.util.MiniClusterWithClientResource) MiniClusterResourceConfiguration(org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration) OperatorState(org.apache.flink.runtime.checkpoint.OperatorState) CheckpointMetadata(org.apache.flink.runtime.checkpoint.metadata.CheckpointMetadata) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) MethodSource(org.junit.jupiter.params.provider.MethodSource)

Aggregations

SavepointFormatType (org.apache.flink.core.execution.SavepointFormatType)11 JobID (org.apache.flink.api.common.JobID)6 IOException (java.io.IOException)5 CompletableFuture (java.util.concurrent.CompletableFuture)5 FlinkException (org.apache.flink.util.FlinkException)5 Test (org.junit.Test)5 List (java.util.List)4 AsynchronousJobOperationKey (org.apache.flink.runtime.rest.handler.job.AsynchronousJobOperationKey)4 TestingRestfulGateway (org.apache.flink.runtime.webmonitor.TestingRestfulGateway)4 ArrayList (java.util.ArrayList)3 Collection (java.util.Collection)3 Collections (java.util.Collections)3 Map (java.util.Map)3 ExecutionException (java.util.concurrent.ExecutionException)3 TimeUnit (java.util.concurrent.TimeUnit)3 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)3 BiConsumer (java.util.function.BiConsumer)3 Predicate (java.util.function.Predicate)3 Supplier (java.util.function.Supplier)3 Collectors (java.util.stream.Collectors)3