Search in sources :

Example 1 with TestingResourceCleanerFactory

use of org.apache.flink.runtime.dispatcher.cleanup.TestingResourceCleanerFactory in project flink by apache.

the class DispatcherResourceCleanupTest method testJobBeingMarkedAsCleanAfterCleanup.

@Test
public void testJobBeingMarkedAsCleanAfterCleanup() throws Exception {
    final CompletableFuture<JobID> markAsCleanFuture = new CompletableFuture<>();
    final JobResultStore jobResultStore = TestingJobResultStore.builder().withMarkResultAsCleanConsumer(markAsCleanFuture::complete).build();
    final OneShotLatch localCleanupLatch = new OneShotLatch();
    final OneShotLatch globalCleanupLatch = new OneShotLatch();
    final TestingResourceCleanerFactory resourceCleanerFactory = TestingResourceCleanerFactory.builder().withLocallyCleanableResource((ignoredJobId, ignoredExecutor) -> {
        try {
            localCleanupLatch.await();
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
        return FutureUtils.completedVoidFuture();
    }).withGloballyCleanableResource((ignoredJobId, ignoredExecutor) -> {
        try {
            globalCleanupLatch.await();
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
        return FutureUtils.completedVoidFuture();
    }).build();
    final TestingDispatcher.Builder dispatcherBuilder = createTestingDispatcherBuilder().setJobResultStore(jobResultStore).setResourceCleanerFactory(resourceCleanerFactory);
    final TestingJobManagerRunnerFactory jobManagerRunnerFactory = startDispatcherAndSubmitJob(dispatcherBuilder, 0);
    finishJob(jobManagerRunnerFactory.takeCreatedJobManagerRunner());
    assertThat(markAsCleanFuture.isDone(), is(false));
    localCleanupLatch.trigger();
    assertThat(markAsCleanFuture.isDone(), is(false));
    globalCleanupLatch.trigger();
    assertThat(markAsCleanFuture.get(), is(jobId));
}
Also used : Arrays(java.util.Arrays) JobSubmissionException(org.apache.flink.runtime.client.JobSubmissionException) ThrowingRunnable(org.apache.flink.util.function.ThrowingRunnable) TestingRpcService(org.apache.flink.runtime.rpc.TestingRpcService) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) TimeoutException(java.util.concurrent.TimeoutException) ExceptionUtils(org.apache.flink.util.ExceptionUtils) JobManagerJobMetricGroupFactory(org.apache.flink.runtime.jobmaster.factories.JobManagerJobMetricGroupFactory) Assert.assertThat(org.junit.Assert.assertThat) After(org.junit.After) TestLogger(org.apache.flink.util.TestLogger) TestingJobMasterGatewayBuilder(org.apache.flink.runtime.jobmaster.utils.TestingJobMasterGatewayBuilder) TestingFatalErrorHandlerResource(org.apache.flink.runtime.util.TestingFatalErrorHandlerResource) Assert.fail(org.junit.Assert.fail) TestingJobManagerRunner(org.apache.flink.runtime.jobmaster.TestingJobManagerRunner) ClassRule(org.junit.ClassRule) AfterClass(org.junit.AfterClass) HighAvailabilityServices(org.apache.flink.runtime.highavailability.HighAvailabilityServices) FlinkMatchers.containsCause(org.apache.flink.core.testutils.FlinkMatchers.containsCause) Preconditions(org.apache.flink.util.Preconditions) Acknowledge(org.apache.flink.runtime.messages.Acknowledge) HeartbeatServices(org.apache.flink.runtime.heartbeat.HeartbeatServices) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) TestingJobResultStore(org.apache.flink.runtime.testutils.TestingJobResultStore) Matchers.equalTo(org.hamcrest.Matchers.equalTo) JobManagerSharedServices(org.apache.flink.runtime.jobmaster.JobManagerSharedServices) Optional(java.util.Optional) AbstractDispatcherTest.awaitStatus(org.apache.flink.runtime.dispatcher.AbstractDispatcherTest.awaitStatus) Matchers.is(org.hamcrest.Matchers.is) JobResultStore(org.apache.flink.runtime.highavailability.JobResultStore) Queue(java.util.Queue) Time(org.apache.flink.api.common.time.Time) OneShotLatch(org.apache.flink.core.testutils.OneShotLatch) FlinkException(org.apache.flink.util.FlinkException) BlobServer(org.apache.flink.runtime.blob.BlobServer) BeforeClass(org.junit.BeforeClass) CompletableFuture(java.util.concurrent.CompletableFuture) JobStatus(org.apache.flink.api.common.JobStatus) TestingJobMasterGateway(org.apache.flink.runtime.jobmaster.utils.TestingJobMasterGateway) TestingResourceCleanerFactory(org.apache.flink.runtime.dispatcher.cleanup.TestingResourceCleanerFactory) RpcService(org.apache.flink.runtime.rpc.RpcService) FutureUtils(org.apache.flink.util.concurrent.FutureUtils) JobGraphTestUtils(org.apache.flink.runtime.jobgraph.JobGraphTestUtils) FatalErrorHandler(org.apache.flink.runtime.rpc.FatalErrorHandler) ExpectedException(org.junit.rules.ExpectedException) IsInstanceOf(org.hamcrest.core.IsInstanceOf) Before(org.junit.Before) ArchivedExecutionGraph(org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph) ArchivedExecutionGraphBuilder(org.apache.flink.runtime.rest.handler.legacy.utils.ArchivedExecutionGraphBuilder) FlinkMatchers.containsMessage(org.apache.flink.core.testutils.FlinkMatchers.containsMessage) Configuration(org.apache.flink.configuration.Configuration) Test(org.junit.Test) IOException(java.io.IOException) TestingBlobStoreBuilder(org.apache.flink.runtime.blob.TestingBlobStoreBuilder) Reference(org.apache.flink.util.Reference) ExecutionGraphInfo(org.apache.flink.runtime.scheduler.ExecutionGraphInfo) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) JobResultEntry(org.apache.flink.runtime.highavailability.JobResultEntry) JobID(org.apache.flink.api.common.JobID) Rule(org.junit.Rule) JobManagerRunner(org.apache.flink.runtime.jobmaster.JobManagerRunner) BlobUtils(org.apache.flink.runtime.blob.BlobUtils) ArrayDeque(java.util.ArrayDeque) DuplicateJobSubmissionException(org.apache.flink.runtime.client.DuplicateJobSubmissionException) TemporaryFolder(org.junit.rules.TemporaryFolder) CompletableFuture(java.util.concurrent.CompletableFuture) OneShotLatch(org.apache.flink.core.testutils.OneShotLatch) TestingResourceCleanerFactory(org.apache.flink.runtime.dispatcher.cleanup.TestingResourceCleanerFactory) TestingJobResultStore(org.apache.flink.runtime.testutils.TestingJobResultStore) JobResultStore(org.apache.flink.runtime.highavailability.JobResultStore) JobID(org.apache.flink.api.common.JobID) Test(org.junit.Test)

Aggregations

IOException (java.io.IOException)1 ArrayDeque (java.util.ArrayDeque)1 Arrays (java.util.Arrays)1 Optional (java.util.Optional)1 Queue (java.util.Queue)1 ArrayBlockingQueue (java.util.concurrent.ArrayBlockingQueue)1 CompletableFuture (java.util.concurrent.CompletableFuture)1 ExecutionException (java.util.concurrent.ExecutionException)1 TimeUnit (java.util.concurrent.TimeUnit)1 TimeoutException (java.util.concurrent.TimeoutException)1 JobID (org.apache.flink.api.common.JobID)1 JobStatus (org.apache.flink.api.common.JobStatus)1 Time (org.apache.flink.api.common.time.Time)1 Configuration (org.apache.flink.configuration.Configuration)1 FlinkMatchers.containsCause (org.apache.flink.core.testutils.FlinkMatchers.containsCause)1 FlinkMatchers.containsMessage (org.apache.flink.core.testutils.FlinkMatchers.containsMessage)1 OneShotLatch (org.apache.flink.core.testutils.OneShotLatch)1 BlobServer (org.apache.flink.runtime.blob.BlobServer)1 BlobUtils (org.apache.flink.runtime.blob.BlobUtils)1 TestingBlobStoreBuilder (org.apache.flink.runtime.blob.TestingBlobStoreBuilder)1