Examples with SharedStateRegistryImpl org.apache.flink.runtime.state.SharedStateRegistryImpl used on opensource projects

Example 11 with SharedStateRegistryImpl

use of org.apache.flink.runtime.state.SharedStateRegistryImpl in project flink by apache.

the class ZooKeeperCompletedCheckpointStoreITCase method testConcurrentCheckpointOperations.

/**
 * FLINK-6612
 *
 * <p>Checks that a concurrent checkpoint completion won't discard a checkpoint which has been
 * recovered by a different completed checkpoint store.
 */
@Test
public void testConcurrentCheckpointOperations() throws Exception {
    final int numberOfCheckpoints = 1;
    final long waitingTimeout = 50L;
    final CompletedCheckpointStore zkCheckpointStore1 = createRecoveredCompletedCheckpointStore(numberOfCheckpoints);
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    TestCompletedCheckpoint completedCheckpoint = createCheckpoint(1, sharedStateRegistry);
    // complete the first checkpoint
    zkCheckpointStore1.addCheckpointAndSubsumeOldestOne(completedCheckpoint, new CheckpointsCleaner(), () -> {
    });
    // recover the checkpoint by a different checkpoint store
    sharedStateRegistry.close();
    sharedStateRegistry = new SharedStateRegistryImpl();
    final CompletedCheckpointStore zkCheckpointStore2 = createRecoveredCompletedCheckpointStore(numberOfCheckpoints);
    CompletedCheckpoint recoveredCheckpoint = zkCheckpointStore2.getLatestCheckpoint();
    assertTrue(recoveredCheckpoint instanceof TestCompletedCheckpoint);
    TestCompletedCheckpoint recoveredTestCheckpoint = (TestCompletedCheckpoint) recoveredCheckpoint;
    // Check that the recovered checkpoint is not yet discarded
    assertFalse(recoveredTestCheckpoint.isDiscarded());
    // complete another checkpoint --> this should remove the first checkpoint from the store
    // because the number of retained checkpoints == 1
    TestCompletedCheckpoint completedCheckpoint2 = createCheckpoint(2, sharedStateRegistry);
    zkCheckpointStore1.addCheckpointAndSubsumeOldestOne(completedCheckpoint2, new CheckpointsCleaner(), () -> {
    });
    List<CompletedCheckpoint> allCheckpoints = zkCheckpointStore1.getAllCheckpoints();
    // check that we have removed the first checkpoint from zkCompletedStore1
    assertEquals(Collections.singletonList(completedCheckpoint2), allCheckpoints);
    // lets wait a little bit to see that no discard operation will be executed
    assertFalse("The checkpoint should not have been discarded.", recoveredTestCheckpoint.awaitDiscard(waitingTimeout));
    // check that we have not discarded the first completed checkpoint
    assertFalse(recoveredTestCheckpoint.isDiscarded());
    TestCompletedCheckpoint completedCheckpoint3 = createCheckpoint(3, sharedStateRegistry);
    // this should release the last lock on completedCheckpoint and thus discard it
    zkCheckpointStore2.addCheckpointAndSubsumeOldestOne(completedCheckpoint3, new CheckpointsCleaner(), () -> {
    });
    // the checkpoint should be discarded eventually because there is no lock on it anymore
    recoveredTestCheckpoint.awaitDiscard();
}

Example 12 with SharedStateRegistryImpl

use of org.apache.flink.runtime.state.SharedStateRegistryImpl in project flink by apache.

the class DefaultCompletedCheckpointStoreTest method createStateHandles.

private List<Tuple2<RetrievableStateHandle<CompletedCheckpoint>, String>> createStateHandles(int num) {
    final List<Tuple2<RetrievableStateHandle<CompletedCheckpoint>, String>> stateHandles = new ArrayList<>();
    for (int i = 1; i <= num; i++) {
        final CompletedCheckpointStoreTest.TestCompletedCheckpoint completedCheckpoint = CompletedCheckpointStoreTest.createCheckpoint(i, new SharedStateRegistryImpl());
        final RetrievableStateHandle<CompletedCheckpoint> checkpointStateHandle = checkpointStorageHelper.store(completedCheckpoint);
        stateHandles.add(new Tuple2<>(checkpointStateHandle, String.valueOf(i)));
    }
    return stateHandles;
}

Example 13 with SharedStateRegistryImpl

use of org.apache.flink.runtime.state.SharedStateRegistryImpl in project flink by apache.

the class DefaultCompletedCheckpointStoreTest method testAddCheckpointFailedShouldNotRemoveOldOnes.

@Test
public void testAddCheckpointFailedShouldNotRemoveOldOnes() throws Exception {
    final int num = 1;
    final String errMsg = "Add to state handle failed.";
    final TestingStateHandleStore<CompletedCheckpoint> stateHandleStore = builder.setGetAllSupplier(() -> createStateHandles(num)).setAddFunction((ignore, ckp) -> {
        throw new FlinkException(errMsg);
    }).build();
    final CompletedCheckpointStore completedCheckpointStore = createCompletedCheckpointStore(stateHandleStore);
    assertThat(completedCheckpointStore.getAllCheckpoints().size(), is(num));
    assertThat(completedCheckpointStore.getAllCheckpoints().get(0).getCheckpointID(), is(1L));
    final long ckpId = 100L;
    final CompletedCheckpoint ckp = CompletedCheckpointStoreTest.createCheckpoint(ckpId, new SharedStateRegistryImpl());
    try {
        completedCheckpointStore.addCheckpointAndSubsumeOldestOne(ckp, new CheckpointsCleaner(), () -> {
        });
        fail("We should get an exception when add checkpoint to failed..");
    } catch (FlinkException ex) {
        assertThat(ex, FlinkMatchers.containsMessage(errMsg));
    }
    // Check the old checkpoint still exists.
    assertThat(completedCheckpointStore.getAllCheckpoints().size(), is(num));
    assertThat(completedCheckpointStore.getAllCheckpoints().get(0).getCheckpointID(), is(1L));
}

Example 14 with SharedStateRegistryImpl

use of org.apache.flink.runtime.state.SharedStateRegistryImpl in project flink by apache.

the class DefaultCompletedCheckpointStoreTest method testAddCheckpointSuccessfullyShouldRemoveOldOnes.

@Test
public void testAddCheckpointSuccessfullyShouldRemoveOldOnes() throws Exception {
    final int num = 1;
    final CompletableFuture<CompletedCheckpoint> addFuture = new CompletableFuture<>();
    final TestingStateHandleStore<CompletedCheckpoint> stateHandleStore = builder.setGetAllSupplier(() -> createStateHandles(num)).setAddFunction((ignore, ckp) -> {
        addFuture.complete(ckp);
        return null;
    }).build();
    final CompletedCheckpointStore completedCheckpointStore = createCompletedCheckpointStore(stateHandleStore);
    assertThat(completedCheckpointStore.getAllCheckpoints().size(), is(num));
    assertThat(completedCheckpointStore.getAllCheckpoints().get(0).getCheckpointID(), is(1L));
    final long ckpId = 100L;
    final CompletedCheckpoint ckp = CompletedCheckpointStoreTest.createCheckpoint(ckpId, new SharedStateRegistryImpl());
    completedCheckpointStore.addCheckpointAndSubsumeOldestOne(ckp, new CheckpointsCleaner(), () -> {
    });
    // We should persist the completed checkpoint to state handle store.
    final CompletedCheckpoint addedCkp = addFuture.get(timeout, TimeUnit.MILLISECONDS);
    assertThat(addedCkp.getCheckpointID(), is(ckpId));
    // Check the old checkpoint is removed and new one is added.
    assertThat(completedCheckpointStore.getAllCheckpoints().size(), is(num));
    assertThat(completedCheckpointStore.getAllCheckpoints().get(0).getCheckpointID(), is(ckpId));
}

Example 15 with SharedStateRegistryImpl

use of org.apache.flink.runtime.state.SharedStateRegistryImpl in project flink by apache.

the class CompletedCheckpointTest method testRegisterStatesAtRegistry.

@Test
public void testRegisterStatesAtRegistry() {
    OperatorState state = mock(OperatorState.class);
    Map<OperatorID, OperatorState> operatorStates = new HashMap<>();
    operatorStates.put(new OperatorID(), state);
    CompletedCheckpoint checkpoint = new CompletedCheckpoint(new JobID(), 0, 0, 1, operatorStates, Collections.emptyList(), CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.RETAIN_ON_FAILURE), new TestCompletedCheckpointStorageLocation());
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    checkpoint.registerSharedStatesAfterRestored(sharedStateRegistry);
    verify(state, times(1)).registerSharedStates(sharedStateRegistry, 0L);
}