Examples with JobStatus org.apache.flink.api.common.JobStatus used on opensource projects

Example 26 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class AbstractQueryableStateTestBase method testWrongJobIdAndWrongQueryableStateName.

/**
 * Tests that the correct exception is thrown if the query contains a wrong jobId or wrong
 * queryable state name.
 */
@Test
@Ignore
public void testWrongJobIdAndWrongQueryableStateName() throws Exception {
    final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT);
    final long numElements = 1024L;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStateBackend(stateBackend);
    env.setParallelism(maxParallelism);
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 1000L));
    DataStream<Tuple2<Integer, Long>> source = env.addSource(new TestAscendingValueSource(numElements));
    ValueStateDescriptor<Tuple2<Integer, Long>> valueState = new ValueStateDescriptor<>("any", source.getType());
    source.keyBy(new KeySelector<Tuple2<Integer, Long>, Integer>() {

        private static final long serialVersionUID = 7662520075515707428L;

        @Override
        public Integer getKey(Tuple2<Integer, Long> value) {
            return value.f0;
        }
    }).asQueryableState("hakuna", valueState);
    try (AutoCancellableJob closableJobGraph = new AutoCancellableJob(deadline, clusterClient, env)) {
        clusterClient.submitJob(closableJobGraph.getJobGraph()).get();
        CompletableFuture<JobStatus> jobStatusFuture = clusterClient.getJobStatus(closableJobGraph.getJobId());
        while (deadline.hasTimeLeft() && !jobStatusFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS).equals(JobStatus.RUNNING)) {
            Thread.sleep(50);
            jobStatusFuture = clusterClient.getJobStatus(closableJobGraph.getJobId());
        }
        assertEquals(JobStatus.RUNNING, jobStatusFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS));
        final JobID wrongJobId = new JobID();
        CompletableFuture<ValueState<Tuple2<Integer, Long>>> unknownJobFuture = client.getKvState(// this is the wrong job id
        wrongJobId, "hakuna", 0, BasicTypeInfo.INT_TYPE_INFO, valueState);
        try {
            unknownJobFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
            // by now the request must have failed.
            fail();
        } catch (ExecutionException e) {
            Assert.assertTrue("GOT: " + e.getCause().getMessage(), e.getCause() instanceof RuntimeException);
            Assert.assertTrue("GOT: " + e.getCause().getMessage(), e.getCause().getMessage().contains("FlinkJobNotFoundException: Could not find Flink job (" + wrongJobId + ")"));
        } catch (Exception f) {
            fail("Unexpected type of exception: " + f.getMessage());
        }
        CompletableFuture<ValueState<Tuple2<Integer, Long>>> unknownQSName = client.getKvState(closableJobGraph.getJobId(), // this is the wrong name.
        "wrong-hakuna", 0, BasicTypeInfo.INT_TYPE_INFO, valueState);
        try {
            unknownQSName.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
            // by now the request must have failed.
            fail();
        } catch (ExecutionException e) {
            Assert.assertTrue("GOT: " + e.getCause().getMessage(), e.getCause() instanceof RuntimeException);
            Assert.assertTrue("GOT: " + e.getCause().getMessage(), e.getCause().getMessage().contains("UnknownKvStateLocation: No KvStateLocation found for KvState instance with name 'wrong-hakuna'."));
        } catch (Exception f) {
            fail("Unexpected type of exception: " + f.getMessage());
        }
    }
}

Example 27 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class HistoryServerArchiveFetcher method convertLegacyJobOverview.

private static String convertLegacyJobOverview(String legacyOverview) throws IOException {
    JsonNode root = mapper.readTree(legacyOverview);
    JsonNode finishedJobs = root.get("finished");
    JsonNode job = finishedJobs.get(0);
    JobID jobId = JobID.fromHexString(job.get("jid").asText());
    String name = job.get("name").asText();
    JobStatus state = JobStatus.valueOf(job.get("state").asText());
    long startTime = job.get("start-time").asLong();
    long endTime = job.get("end-time").asLong();
    long duration = job.get("duration").asLong();
    long lastMod = job.get("last-modification").asLong();
    JsonNode tasks = job.get("tasks");
    int numTasks = tasks.get("total").asInt();
    JsonNode pendingNode = tasks.get("pending");
    // for flink version < 1.4 we have pending field,
    // when version >= 1.4 pending has been split into scheduled, deploying, and created.
    boolean versionLessThan14 = pendingNode != null;
    int created = 0;
    int scheduled;
    int deploying = 0;
    if (versionLessThan14) {
        // pending is a mix of CREATED/SCHEDULED/DEPLOYING
        // to maintain the correct number of task states we pick SCHEDULED
        scheduled = pendingNode.asInt();
    } else {
        created = tasks.get("created").asInt();
        scheduled = tasks.get("scheduled").asInt();
        deploying = tasks.get("deploying").asInt();
    }
    int running = tasks.get("running").asInt();
    int finished = tasks.get("finished").asInt();
    int canceling = tasks.get("canceling").asInt();
    int canceled = tasks.get("canceled").asInt();
    int failed = tasks.get("failed").asInt();
    int[] tasksPerState = new int[ExecutionState.values().length];
    tasksPerState[ExecutionState.CREATED.ordinal()] = created;
    tasksPerState[ExecutionState.SCHEDULED.ordinal()] = scheduled;
    tasksPerState[ExecutionState.DEPLOYING.ordinal()] = deploying;
    tasksPerState[ExecutionState.RUNNING.ordinal()] = running;
    tasksPerState[ExecutionState.FINISHED.ordinal()] = finished;
    tasksPerState[ExecutionState.CANCELING.ordinal()] = canceling;
    tasksPerState[ExecutionState.CANCELED.ordinal()] = canceled;
    tasksPerState[ExecutionState.FAILED.ordinal()] = failed;
    JobDetails jobDetails = new JobDetails(jobId, name, startTime, endTime, duration, state, lastMod, tasksPerState, numTasks);
    MultipleJobsDetails multipleJobsDetails = new MultipleJobsDetails(Collections.singleton(jobDetails));
    StringWriter sw = new StringWriter();
    mapper.writeValue(sw, multipleJobsDetails);
    return sw.toString();
}

Example 28 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class CliFrontend method printJobStatusMessages.

private static void printJobStatusMessages(List<JobStatusMessage> jobs) {
    SimpleDateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss");
    Comparator<JobStatusMessage> startTimeComparator = (o1, o2) -> (int) (o1.getStartTime() - o2.getStartTime());
    Comparator<Map.Entry<JobStatus, List<JobStatusMessage>>> statusComparator = (o1, o2) -> String.CASE_INSENSITIVE_ORDER.compare(o1.getKey().toString(), o2.getKey().toString());
    Map<JobStatus, List<JobStatusMessage>> jobsByState = jobs.stream().collect(Collectors.groupingBy(JobStatusMessage::getJobState));
    jobsByState.entrySet().stream().sorted(statusComparator).map(Map.Entry::getValue).flatMap(List::stream).sorted(startTimeComparator).forEachOrdered(job -> System.out.println(dateFormat.format(new Date(job.getStartTime())) + " : " + job.getJobId() + " : " + job.getJobName() + " (" + job.getJobState() + ")"));
}

Example 29 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class ApplicationDispatcherBootstrapTest method testShutdownDisabled.

@ParameterizedTest
@EnumSource(value = JobStatus.class, names = { "FINISHED", "CANCELED", "FAILED" })
public void testShutdownDisabled(JobStatus jobStatus) throws Exception {
    final Configuration configurationUnderTest = getConfiguration();
    configurationUnderTest.set(DeploymentOptions.SHUTDOWN_ON_APPLICATION_FINISH, false);
    final TestingDispatcherGateway dispatcherGateway = dispatcherGatewayBuilder(jobStatus).setClusterShutdownFunction(status -> {
        fail("Cluster shutdown should not be called");
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).build();
    ApplicationDispatcherBootstrap bootstrap = createApplicationDispatcherBootstrap(configurationUnderTest, dispatcherGateway, scheduledExecutor);
    // Wait until bootstrap is finished to make sure cluster shutdown isn't called
    bootstrap.getBootstrapCompletionFuture().get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
}

Example 30 with JobStatus

use of org.apache.flink.api.common.JobStatus in project flink by apache.

the class DefaultExecutionGraph method allVerticesInTerminalState.

/**
 * This method is a callback during cancellation/failover and called when all tasks have reached
 * a terminal state (cancelled/failed/finished).
 */
private void allVerticesInTerminalState() {
    assertRunningInJobMasterMainThread();
    // we are done, transition to the final state
    JobStatus current;
    while (true) {
        current = this.state;
        if (current == JobStatus.RUNNING) {
            failGlobal(new Exception("ExecutionGraph went into allVerticesInTerminalState() from RUNNING"));
        } else if (current == JobStatus.CANCELLING) {
            if (transitionState(current, JobStatus.CANCELED)) {
                onTerminalState(JobStatus.CANCELED);
                break;
            }
        } else if (current == JobStatus.FAILING) {
            break;
        } else if (current.isGloballyTerminalState()) {
            LOG.warn("Job has entered globally terminal state without waiting for all " + "job vertices to reach final state.");
            break;
        } else {
            failGlobal(new Exception("ExecutionGraph went into final state from state " + current));
            break;
        }
    }
// done transitioning the state
}