Search in sources :

Example 6 with TestingStateHandleStore

use of org.apache.flink.runtime.persistence.TestingStateHandleStore in project flink by apache.

the class DefaultJobGraphStoreTest method testRecoverJobGraph.

@Test
public void testRecoverJobGraph() throws Exception {
    final RetrievableStateHandle<JobGraph> stateHandle = jobGraphStorageHelper.store(testingJobGraph);
    final TestingStateHandleStore<JobGraph> stateHandleStore = builder.setGetFunction(ignore -> stateHandle).build();
    final JobGraphStore jobGraphStore = createAndStartJobGraphStore(stateHandleStore);
    final JobGraph recoveredJobGraph = jobGraphStore.recoverJobGraph(testingJobGraph.getJobID());
    assertThat(recoveredJobGraph, is(notNullValue()));
    assertThat(recoveredJobGraph.getJobID(), is(testingJobGraph.getJobID()));
}
Also used : FlinkException(org.apache.flink.util.FlinkException) Arrays(java.util.Arrays) Tuple3(org.apache.flink.api.java.tuple.Tuple3) FlinkMatchers(org.apache.flink.core.testutils.FlinkMatchers) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) TimeoutException(java.util.concurrent.TimeoutException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) Assert.assertThat(org.junit.Assert.assertThat) After(org.junit.After) Matchers.nullValue(org.hamcrest.Matchers.nullValue) JobGraphTestUtils(org.apache.flink.runtime.jobgraph.JobGraphTestUtils) TestLogger(org.apache.flink.util.TestLogger) StateHandleStore(org.apache.flink.runtime.persistence.StateHandleStore) Assert.fail(org.junit.Assert.fail) Before(org.junit.Before) RetrievableStateHandle(org.apache.flink.runtime.state.RetrievableStateHandle) Matchers.notNullValue(org.hamcrest.Matchers.notNullValue) AbstractID(org.apache.flink.util.AbstractID) Collection(java.util.Collection) Test(org.junit.Test) Collectors(java.util.stream.Collectors) TimeUnit(java.util.concurrent.TimeUnit) Executors(org.apache.flink.util.concurrent.Executors) List(java.util.List) JobID(org.apache.flink.api.common.JobID) TestingRetrievableStateStorageHelper(org.apache.flink.runtime.checkpoint.TestingRetrievableStateStorageHelper) Matchers.contains(org.hamcrest.Matchers.contains) Matchers.is(org.hamcrest.Matchers.is) IntegerResourceVersion(org.apache.flink.runtime.persistence.IntegerResourceVersion) TestingStateHandleStore(org.apache.flink.runtime.persistence.TestingStateHandleStore) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) Test(org.junit.Test)

Example 7 with TestingStateHandleStore

use of org.apache.flink.runtime.persistence.TestingStateHandleStore in project flink by apache.

the class DefaultCompletedCheckpointStoreTest method testShutdownShouldNotDiscardStateHandleWhenJobIsNotGloballyTerminalState.

@Test
public void testShutdownShouldNotDiscardStateHandleWhenJobIsNotGloballyTerminalState() throws Exception {
    final AtomicInteger removeCalledNum = new AtomicInteger(0);
    final CompletableFuture<Void> removeAllFuture = new CompletableFuture<>();
    final CompletableFuture<Void> releaseAllFuture = new CompletableFuture<>();
    final TestingStateHandleStore<CompletedCheckpoint> stateHandleStore = builder.setGetAllSupplier(() -> createStateHandles(3)).setRemoveFunction(ignore -> {
        removeCalledNum.incrementAndGet();
        return true;
    }).setReleaseAllHandlesRunnable(() -> releaseAllFuture.complete(null)).setClearEntriesRunnable(() -> removeAllFuture.complete(null)).build();
    final CompletedCheckpointStore completedCheckpointStore = createCompletedCheckpointStore(stateHandleStore);
    assertThat(completedCheckpointStore.getAllCheckpoints().size(), is(3));
    TestingStreamStateHandle streamStateHandle = registerState(completedCheckpointStore, 3L);
    completedCheckpointStore.shutdown(JobStatus.CANCELLING, new CheckpointsCleaner());
    try {
        removeAllFuture.get(timeout, TimeUnit.MILLISECONDS);
        fail("We should get an expected timeout because the job is not globally terminated.");
    } catch (TimeoutException ex) {
    // expected
    }
    assertThat(removeCalledNum.get(), is(0));
    assertThat(removeAllFuture.isDone(), is(false));
    assertThat(releaseAllFuture.isDone(), is(true));
    assertThat(completedCheckpointStore.getAllCheckpoints().size(), is(0));
    assertThat(streamStateHandle.isDisposed(), is(false));
}
Also used : FlinkException(org.apache.flink.util.FlinkException) SavepointFormatType(org.apache.flink.core.execution.SavepointFormatType) Tuple2(org.apache.flink.api.java.tuple.Tuple2) Assert.assertThrows(org.junit.Assert.assertThrows) FlinkMatchers(org.apache.flink.core.testutils.FlinkMatchers) TestCompletedCheckpointStorageLocation(org.apache.flink.runtime.state.testutils.TestCompletedCheckpointStorageLocation) SharedStateRegistry(org.apache.flink.runtime.state.SharedStateRegistry) TimeoutException(java.util.concurrent.TimeoutException) ExceptionUtils(org.apache.flink.util.ExceptionUtils) CompletableFuture(java.util.concurrent.CompletableFuture) JobStatus(org.apache.flink.api.common.JobStatus) NEVER_RETAIN_AFTER_TERMINATION(org.apache.flink.runtime.checkpoint.CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION) ArrayList(java.util.ArrayList) Assert.assertThat(org.junit.Assert.assertThat) TestingRetrievableStateStorageHelper(org.apache.flink.runtime.persistence.TestingRetrievableStateStorageHelper) SharedStateRegistryKey(org.apache.flink.runtime.state.SharedStateRegistryKey) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Arrays.asList(java.util.Arrays.asList) After(org.junit.After) TestLogger(org.apache.flink.util.TestLogger) StateHandleStore(org.apache.flink.runtime.persistence.StateHandleStore) Assert.fail(org.junit.Assert.fail) ExecutorService(java.util.concurrent.ExecutorService) Before(org.junit.Before) RetrievableStateHandle(org.apache.flink.runtime.state.RetrievableStateHandle) ExecutorThreadFactory(org.apache.flink.util.concurrent.ExecutorThreadFactory) Collections.emptyList(java.util.Collections.emptyList) Test(org.junit.Test) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) TestingStreamStateHandle(org.apache.flink.runtime.state.TestingStreamStateHandle) Executors(java.util.concurrent.Executors) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) JobID(org.apache.flink.api.common.JobID) SharedStateRegistryImpl(org.apache.flink.runtime.state.SharedStateRegistryImpl) Matchers.contains(org.hamcrest.Matchers.contains) Matchers.is(org.hamcrest.Matchers.is) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) TestingStateHandleStore(org.apache.flink.runtime.persistence.TestingStateHandleStore) TestingStreamStateHandle(org.apache.flink.runtime.state.TestingStreamStateHandle) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) TimeoutException(java.util.concurrent.TimeoutException) Test(org.junit.Test)

Example 8 with TestingStateHandleStore

use of org.apache.flink.runtime.persistence.TestingStateHandleStore in project flink by apache.

the class DefaultCompletedCheckpointStoreTest method testAddCheckpointFailedShouldNotRemoveOldOnes.

@Test
public void testAddCheckpointFailedShouldNotRemoveOldOnes() throws Exception {
    final int num = 1;
    final String errMsg = "Add to state handle failed.";
    final TestingStateHandleStore<CompletedCheckpoint> stateHandleStore = builder.setGetAllSupplier(() -> createStateHandles(num)).setAddFunction((ignore, ckp) -> {
        throw new FlinkException(errMsg);
    }).build();
    final CompletedCheckpointStore completedCheckpointStore = createCompletedCheckpointStore(stateHandleStore);
    assertThat(completedCheckpointStore.getAllCheckpoints().size(), is(num));
    assertThat(completedCheckpointStore.getAllCheckpoints().get(0).getCheckpointID(), is(1L));
    final long ckpId = 100L;
    final CompletedCheckpoint ckp = CompletedCheckpointStoreTest.createCheckpoint(ckpId, new SharedStateRegistryImpl());
    try {
        completedCheckpointStore.addCheckpointAndSubsumeOldestOne(ckp, new CheckpointsCleaner(), () -> {
        });
        fail("We should get an exception when add checkpoint to failed..");
    } catch (FlinkException ex) {
        assertThat(ex, FlinkMatchers.containsMessage(errMsg));
    }
    // Check the old checkpoint still exists.
    assertThat(completedCheckpointStore.getAllCheckpoints().size(), is(num));
    assertThat(completedCheckpointStore.getAllCheckpoints().get(0).getCheckpointID(), is(1L));
}
Also used : FlinkException(org.apache.flink.util.FlinkException) SavepointFormatType(org.apache.flink.core.execution.SavepointFormatType) Tuple2(org.apache.flink.api.java.tuple.Tuple2) Assert.assertThrows(org.junit.Assert.assertThrows) FlinkMatchers(org.apache.flink.core.testutils.FlinkMatchers) TestCompletedCheckpointStorageLocation(org.apache.flink.runtime.state.testutils.TestCompletedCheckpointStorageLocation) SharedStateRegistry(org.apache.flink.runtime.state.SharedStateRegistry) TimeoutException(java.util.concurrent.TimeoutException) ExceptionUtils(org.apache.flink.util.ExceptionUtils) CompletableFuture(java.util.concurrent.CompletableFuture) JobStatus(org.apache.flink.api.common.JobStatus) NEVER_RETAIN_AFTER_TERMINATION(org.apache.flink.runtime.checkpoint.CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION) ArrayList(java.util.ArrayList) Assert.assertThat(org.junit.Assert.assertThat) TestingRetrievableStateStorageHelper(org.apache.flink.runtime.persistence.TestingRetrievableStateStorageHelper) SharedStateRegistryKey(org.apache.flink.runtime.state.SharedStateRegistryKey) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Arrays.asList(java.util.Arrays.asList) After(org.junit.After) TestLogger(org.apache.flink.util.TestLogger) StateHandleStore(org.apache.flink.runtime.persistence.StateHandleStore) Assert.fail(org.junit.Assert.fail) ExecutorService(java.util.concurrent.ExecutorService) Before(org.junit.Before) RetrievableStateHandle(org.apache.flink.runtime.state.RetrievableStateHandle) ExecutorThreadFactory(org.apache.flink.util.concurrent.ExecutorThreadFactory) Collections.emptyList(java.util.Collections.emptyList) Test(org.junit.Test) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) TestingStreamStateHandle(org.apache.flink.runtime.state.TestingStreamStateHandle) Executors(java.util.concurrent.Executors) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) JobID(org.apache.flink.api.common.JobID) SharedStateRegistryImpl(org.apache.flink.runtime.state.SharedStateRegistryImpl) Matchers.contains(org.hamcrest.Matchers.contains) Matchers.is(org.hamcrest.Matchers.is) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) TestingStateHandleStore(org.apache.flink.runtime.persistence.TestingStateHandleStore) SharedStateRegistryImpl(org.apache.flink.runtime.state.SharedStateRegistryImpl) FlinkException(org.apache.flink.util.FlinkException) Test(org.junit.Test)

Example 9 with TestingStateHandleStore

use of org.apache.flink.runtime.persistence.TestingStateHandleStore in project flink by apache.

the class DefaultCompletedCheckpointStoreTest method testAddCheckpointSuccessfullyShouldRemoveOldOnes.

@Test
public void testAddCheckpointSuccessfullyShouldRemoveOldOnes() throws Exception {
    final int num = 1;
    final CompletableFuture<CompletedCheckpoint> addFuture = new CompletableFuture<>();
    final TestingStateHandleStore<CompletedCheckpoint> stateHandleStore = builder.setGetAllSupplier(() -> createStateHandles(num)).setAddFunction((ignore, ckp) -> {
        addFuture.complete(ckp);
        return null;
    }).build();
    final CompletedCheckpointStore completedCheckpointStore = createCompletedCheckpointStore(stateHandleStore);
    assertThat(completedCheckpointStore.getAllCheckpoints().size(), is(num));
    assertThat(completedCheckpointStore.getAllCheckpoints().get(0).getCheckpointID(), is(1L));
    final long ckpId = 100L;
    final CompletedCheckpoint ckp = CompletedCheckpointStoreTest.createCheckpoint(ckpId, new SharedStateRegistryImpl());
    completedCheckpointStore.addCheckpointAndSubsumeOldestOne(ckp, new CheckpointsCleaner(), () -> {
    });
    // We should persist the completed checkpoint to state handle store.
    final CompletedCheckpoint addedCkp = addFuture.get(timeout, TimeUnit.MILLISECONDS);
    assertThat(addedCkp.getCheckpointID(), is(ckpId));
    // Check the old checkpoint is removed and new one is added.
    assertThat(completedCheckpointStore.getAllCheckpoints().size(), is(num));
    assertThat(completedCheckpointStore.getAllCheckpoints().get(0).getCheckpointID(), is(ckpId));
}
Also used : FlinkException(org.apache.flink.util.FlinkException) SavepointFormatType(org.apache.flink.core.execution.SavepointFormatType) Tuple2(org.apache.flink.api.java.tuple.Tuple2) Assert.assertThrows(org.junit.Assert.assertThrows) FlinkMatchers(org.apache.flink.core.testutils.FlinkMatchers) TestCompletedCheckpointStorageLocation(org.apache.flink.runtime.state.testutils.TestCompletedCheckpointStorageLocation) SharedStateRegistry(org.apache.flink.runtime.state.SharedStateRegistry) TimeoutException(java.util.concurrent.TimeoutException) ExceptionUtils(org.apache.flink.util.ExceptionUtils) CompletableFuture(java.util.concurrent.CompletableFuture) JobStatus(org.apache.flink.api.common.JobStatus) NEVER_RETAIN_AFTER_TERMINATION(org.apache.flink.runtime.checkpoint.CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION) ArrayList(java.util.ArrayList) Assert.assertThat(org.junit.Assert.assertThat) TestingRetrievableStateStorageHelper(org.apache.flink.runtime.persistence.TestingRetrievableStateStorageHelper) SharedStateRegistryKey(org.apache.flink.runtime.state.SharedStateRegistryKey) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Arrays.asList(java.util.Arrays.asList) After(org.junit.After) TestLogger(org.apache.flink.util.TestLogger) StateHandleStore(org.apache.flink.runtime.persistence.StateHandleStore) Assert.fail(org.junit.Assert.fail) ExecutorService(java.util.concurrent.ExecutorService) Before(org.junit.Before) RetrievableStateHandle(org.apache.flink.runtime.state.RetrievableStateHandle) ExecutorThreadFactory(org.apache.flink.util.concurrent.ExecutorThreadFactory) Collections.emptyList(java.util.Collections.emptyList) Test(org.junit.Test) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) TestingStreamStateHandle(org.apache.flink.runtime.state.TestingStreamStateHandle) Executors(java.util.concurrent.Executors) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) JobID(org.apache.flink.api.common.JobID) SharedStateRegistryImpl(org.apache.flink.runtime.state.SharedStateRegistryImpl) Matchers.contains(org.hamcrest.Matchers.contains) Matchers.is(org.hamcrest.Matchers.is) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) TestingStateHandleStore(org.apache.flink.runtime.persistence.TestingStateHandleStore) CompletableFuture(java.util.concurrent.CompletableFuture) SharedStateRegistryImpl(org.apache.flink.runtime.state.SharedStateRegistryImpl) Test(org.junit.Test)

Aggregations

List (java.util.List)9 CompletableFuture (java.util.concurrent.CompletableFuture)9 TimeUnit (java.util.concurrent.TimeUnit)9 TimeoutException (java.util.concurrent.TimeoutException)9 Collectors (java.util.stream.Collectors)9 JobID (org.apache.flink.api.common.JobID)9 FlinkMatchers (org.apache.flink.core.testutils.FlinkMatchers)9 StateHandleStore (org.apache.flink.runtime.persistence.StateHandleStore)9 TestingStateHandleStore (org.apache.flink.runtime.persistence.TestingStateHandleStore)9 RetrievableStateHandle (org.apache.flink.runtime.state.RetrievableStateHandle)9 FlinkException (org.apache.flink.util.FlinkException)9 TestLogger (org.apache.flink.util.TestLogger)9 Matchers.contains (org.hamcrest.Matchers.contains)9 Matchers.is (org.hamcrest.Matchers.is)9 After (org.junit.After)9 Assert.assertThat (org.junit.Assert.assertThat)9 Assert.fail (org.junit.Assert.fail)9 Before (org.junit.Before)9 Test (org.junit.Test)9 Arrays (java.util.Arrays)4