Examples with JobStatus org.apache.flink.api.common.JobStatus used on opensource projects

Example 21 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class SnapshotMigrationTestBase method restoreAndExecute.

@SafeVarargs
protected final void restoreAndExecute(StreamExecutionEnvironment env, String snapshotPath, Tuple2<String, Integer>... expectedAccumulators) throws Exception {
    final Deadline deadLine = Deadline.fromNow(Duration.ofMinutes(5));
    ClusterClient<?> client = miniClusterResource.getClusterClient();
    // Submit the job
    JobGraph jobGraph = env.getStreamGraph().getJobGraph();
    jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(snapshotPath));
    JobID jobID = client.submitJob(jobGraph).get();
    boolean done = false;
    while (deadLine.hasTimeLeft()) {
        try {
            CompletableFuture<JobStatus> jobStatusFuture = client.getJobStatus(jobID);
            JobStatus jobStatus = jobStatusFuture.get(5, TimeUnit.SECONDS);
            if (jobStatus == JobStatus.FAILED) {
                LOG.warn("Job reached status failed", client.requestJobResult(jobID).get().getSerializedThrowable().get().deserializeError(ClassLoader.getSystemClassLoader()));
            }
            assertNotEquals(JobStatus.FAILED, jobStatus);
        } catch (Exception e) {
            fail("Could not connect to job: " + e);
        }
        Thread.sleep(100);
        Map<String, Object> accumulators = client.getAccumulators(jobID).get();
        boolean allDone = true;
        for (Tuple2<String, Integer> acc : expectedAccumulators) {
            Object numFinished = accumulators.get(acc.f0);
            if (numFinished == null) {
                allDone = false;
                break;
            }
            if (!numFinished.equals(acc.f1)) {
                allDone = false;
                break;
            }
        }
        if (allDone) {
            done = true;
            break;
        }
    }
    if (!done) {
        fail("Did not see the expected accumulator results within time limit.");
    }
}

Example 22 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class WebMonitorMessagesTest method randomJobDetails.

private Collection<JobDetails> randomJobDetails(Random rnd) {
    final JobDetails[] details = new JobDetails[rnd.nextInt(10)];
    for (int k = 0; k < details.length; k++) {
        int[] numVerticesPerState = new int[ExecutionState.values().length];
        int numTotal = 0;
        for (int i = 0; i < numVerticesPerState.length; i++) {
            int count = rnd.nextInt(55);
            numVerticesPerState[i] = count;
            numTotal += count;
        }
        long time = rnd.nextLong();
        long endTime = rnd.nextBoolean() ? -1L : time + rnd.nextInt();
        long lastModified = endTime == -1 ? time + rnd.nextInt() : endTime;
        String name = new GenericMessageTester.StringInstantiator().instantiate(rnd);
        JobID jid = new JobID();
        JobStatus status = JobStatus.values()[rnd.nextInt(JobStatus.values().length)];
        details[k] = new JobDetails(jid, name, time, endTime, endTime - time, status, lastModified, numVerticesPerState, numTotal);
    }
    return Arrays.asList(details);
}

Example 23 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class JobExecutionResultHandlerTest method testCompletedResult.

@Test
public void testCompletedResult() throws Exception {
    final JobStatus jobStatus = JobStatus.FINISHED;
    final ArchivedExecutionGraph executionGraph = new ArchivedExecutionGraphBuilder().setJobID(TEST_JOB_ID).setState(jobStatus).build();
    final TestingRestfulGateway testingRestfulGateway = new TestingRestfulGateway.Builder().setRequestJobStatusFunction(jobId -> {
        assertThat(jobId, equalTo(TEST_JOB_ID));
        return CompletableFuture.completedFuture(jobStatus);
    }).setRequestJobResultFunction(jobId -> {
        assertThat(jobId, equalTo(TEST_JOB_ID));
        return CompletableFuture.completedFuture(JobResult.createFrom(executionGraph));
    }).build();
    final JobExecutionResultResponseBody responseBody = jobExecutionResultHandler.handleRequest(testRequest, testingRestfulGateway).get();
    assertThat(responseBody.getStatus().getId(), equalTo(QueueStatus.Id.COMPLETED));
    assertThat(responseBody.getJobExecutionResult(), not(nullValue()));
}

Example 24 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class AdaptiveSchedulerTest method testJobStatusListenerNotifiedOfJobStatusChanges.

@Test
public void testJobStatusListenerNotifiedOfJobStatusChanges() throws Exception {
    final JobGraph jobGraph = createJobGraph();
    final DefaultDeclarativeSlotPool declarativeSlotPool = createDeclarativeSlotPool(jobGraph.getJobID());
    final Configuration configuration = new Configuration();
    configuration.set(JobManagerOptions.RESOURCE_WAIT_TIMEOUT, Duration.ofMillis(1L));
    final CompletableFuture<Void> jobCreatedNotification = new CompletableFuture<>();
    final CompletableFuture<Void> jobRunningNotification = new CompletableFuture<>();
    final CompletableFuture<Void> jobFinishedNotification = new CompletableFuture<>();
    final CompletableFuture<JobStatus> unexpectedJobStatusNotification = new CompletableFuture<>();
    final AdaptiveScheduler scheduler = new AdaptiveSchedulerBuilder(jobGraph, singleThreadMainThreadExecutor).setJobMasterConfiguration(configuration).setJobStatusListener((jobId, newJobStatus, timestamp) -> {
        switch(newJobStatus) {
            case CREATED:
                jobCreatedNotification.complete(null);
                break;
            case RUNNING:
                jobRunningNotification.complete(null);
                break;
            case FINISHED:
                jobFinishedNotification.complete(null);
                break;
            default:
                unexpectedJobStatusNotification.complete(newJobStatus);
        }
    }).setDeclarativeSlotPool(declarativeSlotPool).build();
    final SubmissionBufferingTaskManagerGateway taskManagerGateway = new SubmissionBufferingTaskManagerGateway(1 + PARALLELISM);
    singleThreadMainThreadExecutor.execute(() -> {
        scheduler.startScheduling();
        offerSlots(declarativeSlotPool, createSlotOffersForResourceRequirements(ResourceCounter.withResource(ResourceProfile.UNKNOWN, 1)), taskManagerGateway);
    });
    // wait for the task submission
    final TaskDeploymentDescriptor submittedTask = taskManagerGateway.submittedTasks.take();
    // let the job finish
    singleThreadMainThreadExecutor.execute(() -> scheduler.updateTaskExecutionState(new TaskExecutionState(submittedTask.getExecutionAttemptId(), ExecutionState.FINISHED)));
    jobCreatedNotification.get();
    jobRunningNotification.get();
    jobFinishedNotification.get();
    assertThat(unexpectedJobStatusNotification.isDone()).isFalse();
}

Example 25 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class CancelingTestBase method runAndCancelJob.

// --------------------------------------------------------------------------------------------
protected void runAndCancelJob(Plan plan, final int msecsTillCanceling, int maxTimeTillCanceled) throws Exception {
    // submit job
    final JobGraph jobGraph = getJobGraph(plan);
    final long rpcTimeout = configuration.get(AkkaOptions.ASK_TIMEOUT_DURATION).toMillis();
    ClusterClient<?> client = CLUSTER.getClusterClient();
    JobID jobID = client.submitJob(jobGraph).get();
    Deadline submissionDeadLine = new FiniteDuration(2, TimeUnit.MINUTES).fromNow();
    JobStatus jobStatus = client.getJobStatus(jobID).get(rpcTimeout, TimeUnit.MILLISECONDS);
    while (jobStatus != JobStatus.RUNNING && submissionDeadLine.hasTimeLeft()) {
        Thread.sleep(50);
        jobStatus = client.getJobStatus(jobID).get(rpcTimeout, TimeUnit.MILLISECONDS);
    }
    if (jobStatus != JobStatus.RUNNING) {
        Assert.fail("Job not in state RUNNING.");
    }
    Thread.sleep(msecsTillCanceling);
    client.cancel(jobID).get();
    Deadline cancelDeadline = new FiniteDuration(maxTimeTillCanceled, TimeUnit.MILLISECONDS).fromNow();
    JobStatus jobStatusAfterCancel = client.getJobStatus(jobID).get(rpcTimeout, TimeUnit.MILLISECONDS);
    while (jobStatusAfterCancel != JobStatus.CANCELED && cancelDeadline.hasTimeLeft()) {
        Thread.sleep(50);
        jobStatusAfterCancel = client.getJobStatus(jobID).get(rpcTimeout, TimeUnit.MILLISECONDS);
    }
    assertEquals(JobStatus.CANCELED, jobStatusAfterCancel);
}