Search in sources :

Example 6 with SharedStateRegistry

use of org.apache.flink.runtime.state.SharedStateRegistry in project flink by apache.

the class ZooKeeperCompletedCheckpointStoreITCase method testSuspendKeepsCheckpoints.

/**
 * Tests that suspends keeps all checkpoints (so that they can be recovered later by the
 * ZooKeeper store). Furthermore, suspending a job should release all locks.
 */
@Test
public void testSuspendKeepsCheckpoints() throws Exception {
    CuratorFramework client = ZOOKEEPER.getClient();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CompletedCheckpointStore store = createRecoveredCompletedCheckpointStore(1);
    TestCompletedCheckpoint checkpoint = createCheckpoint(0, sharedStateRegistry);
    store.addCheckpointAndSubsumeOldestOne(checkpoint, new CheckpointsCleaner(), () -> {
    });
    assertEquals(1, store.getNumberOfRetainedCheckpoints());
    assertNotNull(client.checkExists().forPath(CHECKPOINT_PATH + checkpointStoreUtil.checkpointIDToName(checkpoint.getCheckpointID())));
    store.shutdown(JobStatus.SUSPENDED, new CheckpointsCleaner());
    assertEquals(0, store.getNumberOfRetainedCheckpoints());
    final String checkpointPath = CHECKPOINT_PATH + checkpointStoreUtil.checkpointIDToName(checkpoint.getCheckpointID());
    Stat stat = client.checkExists().forPath(checkpointPath);
    assertNotNull("The checkpoint node should exist.", stat);
    assertEquals("The checkpoint node should not be locked.", 0, stat.getNumChildren());
    // Recover again
    sharedStateRegistry.close();
    store = createRecoveredCompletedCheckpointStore(1);
    CompletedCheckpoint recovered = store.getLatestCheckpoint();
    assertEquals(checkpoint, recovered);
}
Also used : CuratorFramework(org.apache.flink.shaded.curator5.org.apache.curator.framework.CuratorFramework) Stat(org.apache.flink.shaded.zookeeper3.org.apache.zookeeper.data.Stat) SharedStateRegistryImpl(org.apache.flink.runtime.state.SharedStateRegistryImpl) SharedStateRegistry(org.apache.flink.runtime.state.SharedStateRegistry) Test(org.junit.Test)

Example 7 with SharedStateRegistry

use of org.apache.flink.runtime.state.SharedStateRegistry in project flink by apache.

the class ZooKeeperCompletedCheckpointStoreITCase method testLatestCheckpointRecovery.

/**
 * FLINK-6284.
 *
 * <p>Tests that the latest recovered checkpoint is the one with the highest checkpoint id
 */
@Test
public void testLatestCheckpointRecovery() throws Exception {
    final int numCheckpoints = 3;
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CompletedCheckpointStore checkpointStore = createRecoveredCompletedCheckpointStore(numCheckpoints);
    List<CompletedCheckpoint> checkpoints = new ArrayList<>(numCheckpoints);
    checkpoints.add(createCheckpoint(9, sharedStateRegistry));
    checkpoints.add(createCheckpoint(10, sharedStateRegistry));
    checkpoints.add(createCheckpoint(11, sharedStateRegistry));
    for (CompletedCheckpoint checkpoint : checkpoints) {
        checkpointStore.addCheckpointAndSubsumeOldestOne(checkpoint, new CheckpointsCleaner(), () -> {
        });
    }
    sharedStateRegistry.close();
    final CompletedCheckpoint latestCheckpoint = createRecoveredCompletedCheckpointStore(numCheckpoints).getLatestCheckpoint();
    assertEquals(checkpoints.get(checkpoints.size() - 1), latestCheckpoint);
}
Also used : SharedStateRegistryImpl(org.apache.flink.runtime.state.SharedStateRegistryImpl) ArrayList(java.util.ArrayList) CheckpointRequestDeciderTest.regularCheckpoint(org.apache.flink.runtime.checkpoint.CheckpointRequestDeciderTest.regularCheckpoint) SharedStateRegistry(org.apache.flink.runtime.state.SharedStateRegistry) Test(org.junit.Test)

Example 8 with SharedStateRegistry

use of org.apache.flink.runtime.state.SharedStateRegistry in project flink by apache.

the class CompletedCheckpointTest method testCleanUpOnSubsume.

/**
 * Tests that the garbage collection properties are respected when subsuming checkpoints.
 */
@Test
public void testCleanUpOnSubsume() throws Exception {
    OperatorState state = mock(OperatorState.class);
    Map<OperatorID, OperatorState> operatorStates = new HashMap<>();
    operatorStates.put(new OperatorID(), state);
    EmptyStreamStateHandle metadata = new EmptyStreamStateHandle();
    TestCompletedCheckpointStorageLocation location = new TestCompletedCheckpointStorageLocation(metadata, "ptr");
    CheckpointProperties props = new CheckpointProperties(false, CheckpointType.CHECKPOINT, true, false, false, false, false, false);
    CompletedCheckpoint checkpoint = new CompletedCheckpoint(new JobID(), 0, 0, 1, operatorStates, Collections.emptyList(), props, location);
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    checkpoint.registerSharedStatesAfterRestored(sharedStateRegistry);
    verify(state, times(1)).registerSharedStates(sharedStateRegistry, 0L);
    // Subsume
    checkpoint.discardOnSubsume();
    verify(state, times(1)).discardState();
    assertTrue(location.isDisposed());
    assertTrue(metadata.isDisposed());
}
Also used : EmptyStreamStateHandle(org.apache.flink.runtime.state.testutils.EmptyStreamStateHandle) HashMap(java.util.HashMap) SharedStateRegistryImpl(org.apache.flink.runtime.state.SharedStateRegistryImpl) TestCompletedCheckpointStorageLocation(org.apache.flink.runtime.state.testutils.TestCompletedCheckpointStorageLocation) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) JobID(org.apache.flink.api.common.JobID) SharedStateRegistry(org.apache.flink.runtime.state.SharedStateRegistry) Test(org.junit.Test)

Example 9 with SharedStateRegistry

use of org.apache.flink.runtime.state.SharedStateRegistry in project flink by apache.

the class StandaloneCompletedCheckpointStoreTest method testShutdownDiscardsCheckpoints.

/**
 * Tests that shutdown discards all checkpoints.
 */
@Test
public void testShutdownDiscardsCheckpoints() throws Exception {
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CompletedCheckpointStore store = createRecoveredCompletedCheckpointStore(1);
    TestCompletedCheckpoint checkpoint = createCheckpoint(0, sharedStateRegistry);
    Collection<OperatorState> operatorStates = checkpoint.getOperatorStates().values();
    store.addCheckpointAndSubsumeOldestOne(checkpoint, new CheckpointsCleaner(), () -> {
    });
    assertEquals(1, store.getNumberOfRetainedCheckpoints());
    verifyCheckpointRegistered(operatorStates, sharedStateRegistry);
    store.shutdown(JobStatus.FINISHED, new CheckpointsCleaner());
    assertEquals(0, store.getNumberOfRetainedCheckpoints());
    assertTrue(checkpoint.isDiscarded());
    verifyCheckpointDiscarded(operatorStates);
}
Also used : SharedStateRegistryImpl(org.apache.flink.runtime.state.SharedStateRegistryImpl) SharedStateRegistry(org.apache.flink.runtime.state.SharedStateRegistry) Test(org.junit.Test)

Example 10 with SharedStateRegistry

use of org.apache.flink.runtime.state.SharedStateRegistry in project flink by apache.

the class SchedulerUtilsTest method testSharedStateRegistration.

/**
 * Check that a {@link SharedStateRegistryFactory} used by {@link SchedulerUtils} registers
 * shared checkpoint state on restore.
 */
@Test
public void testSharedStateRegistration() throws Exception {
    UUID backendId = UUID.randomUUID();
    StateHandleID key = new StateHandleID("k0");
    StreamStateHandle handle = new ByteStreamStateHandle("h0", new byte[] { 1, 2, 3 });
    CheckpointRecoveryFactory recoveryFactory = buildRecoveryFactory(buildCheckpoint(buildIncrementalHandle(key, handle, backendId)));
    CompletedCheckpointStore checkpointStore = SchedulerUtils.createCompletedCheckpointStore(new Configuration(), recoveryFactory, Executors.directExecutor(), log, new JobID());
    SharedStateRegistry sharedStateRegistry = checkpointStore.getSharedStateRegistry();
    IncrementalRemoteKeyedStateHandle newHandle = buildIncrementalHandle(key, new PlaceholderStreamStateHandle(1L), backendId);
    newHandle.registerSharedStates(sharedStateRegistry, 1L);
    assertSame(handle, newHandle.getSharedState().get(key));
}
Also used : PlaceholderStreamStateHandle(org.apache.flink.runtime.state.PlaceholderStreamStateHandle) StreamStateHandle(org.apache.flink.runtime.state.StreamStateHandle) PlaceholderStreamStateHandle(org.apache.flink.runtime.state.PlaceholderStreamStateHandle) ByteStreamStateHandle(org.apache.flink.runtime.state.memory.ByteStreamStateHandle) Configuration(org.apache.flink.configuration.Configuration) StateHandleID(org.apache.flink.runtime.state.StateHandleID) IncrementalRemoteKeyedStateHandle(org.apache.flink.runtime.state.IncrementalRemoteKeyedStateHandle) ByteStreamStateHandle(org.apache.flink.runtime.state.memory.ByteStreamStateHandle) UUID(java.util.UUID) CheckpointRecoveryFactory(org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory) StandaloneCheckpointRecoveryFactory(org.apache.flink.runtime.checkpoint.StandaloneCheckpointRecoveryFactory) JobID(org.apache.flink.api.common.JobID) CompletedCheckpointStore(org.apache.flink.runtime.checkpoint.CompletedCheckpointStore) EmbeddedCompletedCheckpointStore(org.apache.flink.runtime.checkpoint.EmbeddedCompletedCheckpointStore) SharedStateRegistry(org.apache.flink.runtime.state.SharedStateRegistry) Test(org.junit.Test)

Aggregations

SharedStateRegistry (org.apache.flink.runtime.state.SharedStateRegistry)22 SharedStateRegistryImpl (org.apache.flink.runtime.state.SharedStateRegistryImpl)19 Test (org.junit.Test)19 ArrayList (java.util.ArrayList)5 HashMap (java.util.HashMap)5 KeyedStateHandle (org.apache.flink.runtime.state.KeyedStateHandle)5 JobID (org.apache.flink.api.common.JobID)3 Configuration (org.apache.flink.configuration.Configuration)3 OperatorID (org.apache.flink.runtime.jobgraph.OperatorID)3 IncrementalRemoteKeyedStateHandle (org.apache.flink.runtime.state.IncrementalRemoteKeyedStateHandle)3 StateHandleID (org.apache.flink.runtime.state.StateHandleID)3 StreamStateHandle (org.apache.flink.runtime.state.StreamStateHandle)3 TestCompletedCheckpointStorageLocation (org.apache.flink.runtime.state.testutils.TestCompletedCheckpointStorageLocation)3 HashSet (java.util.HashSet)2 Map (java.util.Map)2 CompletableFuture (java.util.concurrent.CompletableFuture)2 ValueStateDescriptor (org.apache.flink.api.common.state.ValueStateDescriptor)2 CheckpointCoordinatorBuilder (org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.CheckpointCoordinatorBuilder)2 CheckpointRequestDeciderTest.regularCheckpoint (org.apache.flink.runtime.checkpoint.CheckpointRequestDeciderTest.regularCheckpoint)2 ExecutionGraph (org.apache.flink.runtime.executiongraph.ExecutionGraph)2