Search in sources :

Example 61 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class CommonTestUtils method waitForAllTaskRunning.

public static void waitForAllTaskRunning(SupplierWithException<AccessExecutionGraph, Exception> executionGraphSupplier, Deadline timeout, boolean allowFinished) throws Exception {
    Predicate<AccessExecutionVertex> subtaskPredicate = task -> {
        switch(task.getExecutionState()) {
            case RUNNING:
                return true;
            case FINISHED:
                if (allowFinished) {
                    return true;
                } else {
                    throw new RuntimeException("Sub-Task finished unexpectedly" + task);
                }
            default:
                return false;
        }
    };
    waitUntilCondition(() -> {
        final AccessExecutionGraph graph = executionGraphSupplier.get();
        if (graph.getState().isGloballyTerminalState()) {
            final ErrorInfo failureInfo = graph.getFailureInfo();
            fail(format("Graph is in globally terminal state (%s)", graph.getState()), failureInfo != null ? failureInfo.getException() : null);
        }
        return graph.getState() == JobStatus.RUNNING && graph.getAllVertices().values().stream().allMatch(jobVertex -> Arrays.stream(jobVertex.getTaskVertices()).allMatch(subtaskPredicate));
    }, timeout);
}
Also used : Assertions.fail(org.junit.jupiter.api.Assertions.fail) Deadline(org.apache.flink.api.common.time.Deadline) Arrays(java.util.Arrays) BufferedInputStream(java.io.BufferedInputStream) FileUtils(org.apache.flink.util.FileUtils) AccessExecutionVertex(org.apache.flink.runtime.executiongraph.AccessExecutionVertex) TimeoutException(java.util.concurrent.TimeoutException) JobStatus(org.apache.flink.api.common.JobStatus) JobVertexID(org.apache.flink.runtime.jobgraph.JobVertexID) Duration(java.time.Duration) Map(java.util.Map) MiniCluster(org.apache.flink.runtime.minicluster.MiniCluster) ManagementFactory(java.lang.management.ManagementFactory) PrintWriter(java.io.PrintWriter) RuntimeMXBean(java.lang.management.RuntimeMXBean) Predicate(java.util.function.Predicate) JobDetailsInfo(org.apache.flink.runtime.rest.messages.job.JobDetailsInfo) StringWriter(java.io.StringWriter) Collection(java.util.Collection) ExecutionState(org.apache.flink.runtime.execution.ExecutionState) FileWriter(java.io.FileWriter) ErrorInfo(org.apache.flink.runtime.executiongraph.ErrorInfo) IOException(java.io.IOException) JobClient(org.apache.flink.core.execution.JobClient) File(java.io.File) String.format(java.lang.String.format) AccessExecutionGraph(org.apache.flink.runtime.executiongraph.AccessExecutionGraph) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) JobID(org.apache.flink.api.common.JobID) ChronoUnit(java.time.temporal.ChronoUnit) Stream(java.util.stream.Stream) SupplierWithException(org.apache.flink.util.function.SupplierWithException) InputStream(java.io.InputStream) ErrorInfo(org.apache.flink.runtime.executiongraph.ErrorInfo) AccessExecutionGraph(org.apache.flink.runtime.executiongraph.AccessExecutionGraph) AccessExecutionVertex(org.apache.flink.runtime.executiongraph.AccessExecutionVertex)

Example 62 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class SystemProcessingTimeService method shutdownServiceUninterruptible.

@Override
public boolean shutdownServiceUninterruptible(long timeoutMs) {
    final Deadline deadline = Deadline.fromNow(Duration.ofMillis(timeoutMs));
    boolean shutdownComplete = false;
    boolean receivedInterrupt = false;
    do {
        try {
            // wait for a reasonable time for all pending timer threads to finish
            shutdownComplete = shutdownAndAwaitPending(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
        } catch (InterruptedException iex) {
            receivedInterrupt = true;
            LOG.trace("Intercepted attempt to interrupt timer service shutdown.", iex);
        }
    } while (deadline.hasTimeLeft() && !shutdownComplete);
    if (receivedInterrupt) {
        Thread.currentThread().interrupt();
    }
    return shutdownComplete;
}
Also used : Deadline(org.apache.flink.api.common.time.Deadline)

Example 63 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class AbstractHAJobRunITCase method testJobExecutionInHaMode.

@Test
public void testJobExecutionInHaMode(@InjectMiniCluster MiniCluster flinkCluster) throws Exception {
    final JobGraph jobGraph = JobGraphTestUtils.singleNoOpJobGraph();
    // providing a timeout helps making the test fail in case some issue occurred while
    // initializing the cluster
    flinkCluster.submitJob(jobGraph).get(30, TimeUnit.SECONDS);
    final Deadline deadline = Deadline.fromNow(Duration.ofSeconds(30));
    final JobStatus jobStatus = FutureUtils.retrySuccessfulWithDelay(() -> flinkCluster.getJobStatus(jobGraph.getJobID()), Time.milliseconds(10), deadline, status -> flinkCluster.isRunning() && status == JobStatus.FINISHED, TestingUtils.defaultScheduledExecutor()).get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
    assertThat(jobStatus).isEqualTo(JobStatus.FINISHED);
    runAfterJobTermination();
}
Also used : JobStatus(org.apache.flink.api.common.JobStatus) Deadline(org.apache.flink.api.common.time.Deadline) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) Configuration(org.apache.flink.configuration.Configuration) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) JobStatus(org.apache.flink.api.common.JobStatus) Order(org.junit.jupiter.api.Order) Test(org.junit.jupiter.api.Test) TimeUnit(java.util.concurrent.TimeUnit) TestingUtils(org.apache.flink.testutils.TestingUtils) TestLoggerExtension(org.apache.flink.util.TestLoggerExtension) AllCallbackWrapper(org.apache.flink.core.testutils.AllCallbackWrapper) FileSystem(org.apache.flink.core.fs.FileSystem) FutureUtils(org.apache.flink.util.concurrent.FutureUtils) ExtendWith(org.junit.jupiter.api.extension.ExtendWith) RegisterExtension(org.junit.jupiter.api.extension.RegisterExtension) InjectMiniCluster(org.apache.flink.test.junit5.InjectMiniCluster) Duration(java.time.Duration) JobGraphTestUtils(org.apache.flink.runtime.jobgraph.JobGraphTestUtils) MiniCluster(org.apache.flink.runtime.minicluster.MiniCluster) Time(org.apache.flink.api.common.time.Time) HighAvailabilityOptions(org.apache.flink.configuration.HighAvailabilityOptions) ZooKeeperExtension(org.apache.flink.runtime.zookeeper.ZooKeeperExtension) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) Deadline(org.apache.flink.api.common.time.Deadline) Test(org.junit.jupiter.api.Test)

Example 64 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class FileChannelManagerImplTest method testDirectoriesCleanupOnKill.

private void testDirectoriesCleanupOnKill(boolean callerHasHook) throws Exception {
    assumeTrue(OperatingSystem.isLinux() || OperatingSystem.isFreeBSD() || OperatingSystem.isSolaris() || OperatingSystem.isMac());
    File fileChannelDir = temporaryFolder.newFolder();
    File signalDir = temporaryFolder.newFolder();
    File signalFile = new File(signalDir.getAbsolutePath(), SIGNAL_FILE_FOR_KILLING);
    FileChannelManagerTestProcess fileChannelManagerTestProcess = new FileChannelManagerTestProcess(callerHasHook, fileChannelDir.getAbsolutePath(), signalFile.getAbsolutePath());
    try {
        fileChannelManagerTestProcess.startProcess();
        // Waits till the process has created temporary files and registered the corresponding
        // shutdown hooks.
        TestJvmProcess.waitForMarkerFile(signalFile, TEST_TIMEOUT.toMillis());
        Process kill = Runtime.getRuntime().exec("kill " + fileChannelManagerTestProcess.getProcessId());
        kill.waitFor();
        assertEquals("Failed to send SIG_TERM to process", 0, kill.exitValue());
        Deadline deadline = Deadline.now().plus(TEST_TIMEOUT);
        while (fileChannelManagerTestProcess.isAlive() && deadline.hasTimeLeft()) {
            Thread.sleep(100);
        }
        assertFalse("The file channel manager test process does not terminate in time, its output is: \n" + fileChannelManagerTestProcess.getProcessOutput(), fileChannelManagerTestProcess.isAlive());
        // Checks if the directories are cleared.
        assertFalse("The file channel manager test process does not remove the tmp shuffle directories after termination, " + "its output is \n" + fileChannelManagerTestProcess.getProcessOutput(), fileOrDirExists(fileChannelDir, DIR_NAME_PREFIX));
    } finally {
        fileChannelManagerTestProcess.destroy();
    }
}
Also used : Deadline(org.apache.flink.api.common.time.Deadline) TestJvmProcess(org.apache.flink.runtime.testutils.TestJvmProcess) File(java.io.File)

Example 65 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class LeaderChangeClusterComponentsTest method testTaskExecutorsReconnectToClusterWithLeadershipChange.

@Test
public void testTaskExecutorsReconnectToClusterWithLeadershipChange() throws Exception {
    final Deadline deadline = Deadline.fromNow(TESTING_TIMEOUT);
    waitUntilTaskExecutorsHaveConnected(NUM_TMS, deadline);
    highAvailabilityServices.revokeResourceManagerLeadership().get();
    highAvailabilityServices.grantResourceManagerLeadership();
    // wait for the ResourceManager to confirm the leadership
    assertThat(LeaderRetrievalUtils.retrieveLeaderConnectionInfo(highAvailabilityServices.getResourceManagerLeaderRetriever(), TESTING_TIMEOUT).getLeaderSessionId(), is(notNullValue()));
    waitUntilTaskExecutorsHaveConnected(NUM_TMS, deadline);
}
Also used : Deadline(org.apache.flink.api.common.time.Deadline) Test(org.junit.Test)

Aggregations

Deadline (org.apache.flink.api.common.time.Deadline)75 Test (org.junit.Test)34 JobID (org.apache.flink.api.common.JobID)29 JobGraph (org.apache.flink.runtime.jobgraph.JobGraph)26 Duration (java.time.Duration)19 Configuration (org.apache.flink.configuration.Configuration)15 StreamExecutionEnvironment (org.apache.flink.streaming.api.environment.StreamExecutionEnvironment)14 Tuple2 (org.apache.flink.api.java.tuple.Tuple2)13 IOException (java.io.IOException)12 ExecutionException (java.util.concurrent.ExecutionException)12 KeySelector (org.apache.flink.api.java.functions.KeySelector)12 AtomicLong (java.util.concurrent.atomic.AtomicLong)11 MiniCluster (org.apache.flink.runtime.minicluster.MiniCluster)10 File (java.io.File)9 TimeUnit (java.util.concurrent.TimeUnit)9 JobStatus (org.apache.flink.api.common.JobStatus)9 List (java.util.List)8 Test (org.junit.jupiter.api.Test)8 CompletableFuture (java.util.concurrent.CompletableFuture)7 CountDownLatch (java.util.concurrent.CountDownLatch)7