Examples with JobDetailsInfo org.apache.flink.runtime.rest.messages.job.JobDetailsInfo used on opensource projects

Example 1 with JobDetailsInfo

use of org.apache.flink.runtime.rest.messages.job.JobDetailsInfo in project flink by apache.

the class JobDetailsHandler method createJobDetailsInfo.

private static JobDetailsInfo createJobDetailsInfo(AccessExecutionGraph executionGraph, @Nullable MetricFetcher metricFetcher) {
    final long now = System.currentTimeMillis();
    final long startTime = executionGraph.getStatusTimestamp(JobStatus.INITIALIZING);
    final long endTime = executionGraph.getState().isGloballyTerminalState() ? executionGraph.getStatusTimestamp(executionGraph.getState()) : -1L;
    final long duration = (endTime > 0L ? endTime : now) - startTime;
    final Map<JobStatus, Long> timestamps = new HashMap<>(JobStatus.values().length);
    for (JobStatus jobStatus : JobStatus.values()) {
        timestamps.put(jobStatus, executionGraph.getStatusTimestamp(jobStatus));
    }
    Collection<JobDetailsInfo.JobVertexDetailsInfo> jobVertexInfos = new ArrayList<>(executionGraph.getAllVertices().size());
    int[] jobVerticesPerState = new int[ExecutionState.values().length];
    for (AccessExecutionJobVertex accessExecutionJobVertex : executionGraph.getVerticesTopologically()) {
        final JobDetailsInfo.JobVertexDetailsInfo vertexDetailsInfo = createJobVertexDetailsInfo(accessExecutionJobVertex, now, executionGraph.getJobID(), metricFetcher);
        jobVertexInfos.add(vertexDetailsInfo);
        jobVerticesPerState[vertexDetailsInfo.getExecutionState().ordinal()]++;
    }
    Map<ExecutionState, Integer> jobVerticesPerStateMap = new HashMap<>(ExecutionState.values().length);
    for (ExecutionState executionState : ExecutionState.values()) {
        jobVerticesPerStateMap.put(executionState, jobVerticesPerState[executionState.ordinal()]);
    }
    return new JobDetailsInfo(executionGraph.getJobID(), executionGraph.getJobName(), executionGraph.isStoppable(), executionGraph.getState(), startTime, endTime, duration, executionGraph.getArchivedExecutionConfig().getMaxParallelism(), now, timestamps, jobVertexInfos, jobVerticesPerStateMap, executionGraph.getJsonPlan());
}

Example 2 with JobDetailsInfo

use of org.apache.flink.runtime.rest.messages.job.JobDetailsInfo in project flink by apache.

the class RestClusterClientTest method testNotShowSuspendedJobStatus.

/**
 * The SUSPENDED job status should never be returned by the client thus client retries until it
 * either receives a different job status or the cluster is not reachable.
 */
@Test
public void testNotShowSuspendedJobStatus() throws Exception {
    final List<JobDetailsInfo> jobDetails = new ArrayList<>();
    jobDetails.add(buildJobDetail(JobStatus.SUSPENDED));
    jobDetails.add(buildJobDetail(JobStatus.RUNNING));
    final TestJobStatusHandler jobStatusHandler = new TestJobStatusHandler(jobDetails.iterator());
    try (TestRestServerEndpoint restServerEndpoint = createRestServerEndpoint(jobStatusHandler)) {
        final RestClusterClient<?> restClusterClient = createRestClusterClient(restServerEndpoint.getServerAddress().getPort());
        try {
            final CompletableFuture<JobStatus> future = restClusterClient.getJobStatus(jobId);
            assertEquals(JobStatus.RUNNING, future.get());
        } finally {
            restClusterClient.close();
        }
    }
}

Example 3 with JobDetailsInfo

use of org.apache.flink.runtime.rest.messages.job.JobDetailsInfo in project flink by apache.

the class SavepointTestBase method waitForAllRunningOrSomeTerminal.

public static void waitForAllRunningOrSomeTerminal(JobID jobID, MiniClusterWithClientResource miniClusterResource) throws Exception {
    while (true) {
        JobDetailsInfo jobInfo = miniClusterResource.getRestClusterClient().getJobDetails(jobID).get();
        Set<ExecutionState> vertexStates = jobInfo.getJobVertexInfos().stream().map(JobDetailsInfo.JobVertexDetailsInfo::getExecutionState).collect(Collectors.toSet());
        if (vertexStates.equals(EnumSet.of(RUNNING)) || vertexStates.stream().anyMatch(ExecutionState::isTerminal)) {
            return;
        } else {
            Thread.sleep(500);
        }
    }
}

Example 4 with JobDetailsInfo

use of org.apache.flink.runtime.rest.messages.job.JobDetailsInfo in project flink by apache.

the class CommonTestUtils method waitForAllTaskRunning.

public static void waitForAllTaskRunning(SupplierWithException<JobDetailsInfo, Exception> jobDetailsSupplier, Deadline timeout) throws Exception {
    waitUntilCondition(() -> {
        final JobDetailsInfo jobDetailsInfo = jobDetailsSupplier.get();
        final Collection<JobDetailsInfo.JobVertexDetailsInfo> vertexInfos = jobDetailsInfo.getJobVertexInfos();
        if (vertexInfos.size() == 0) {
            return false;
        }
        for (JobDetailsInfo.JobVertexDetailsInfo vertexInfo : vertexInfos) {
            final Integer numRunningTasks = vertexInfo.getTasksPerState().get(ExecutionState.RUNNING);
            if (numRunningTasks == null || numRunningTasks != vertexInfo.getParallelism()) {
                return false;
            }
        }
        return true;
    }, timeout, "Some tasks are not running until timeout");
}

Example 5 with JobDetailsInfo

use of org.apache.flink.runtime.rest.messages.job.JobDetailsInfo in project flink by apache.

the class MetricQuerier method getAggregatedMetricsByRestAPI.

public Double getAggregatedMetricsByRestAPI(TestEnvironment.Endpoint endpoint, JobID jobId, String sourceOrSinkName, String metricName, String filter) throws Exception {
    // get job details, including the vertex id
    JobDetailsInfo jobDetailsInfo = getJobDetails(restClient, endpoint, jobId);
    // get the vertex id for source/sink operator
    JobDetailsInfo.JobVertexDetailsInfo vertex = jobDetailsInfo.getJobVertexInfos().stream().filter(v -> v.getName().contains(sourceOrSinkName)).findAny().orElse(null);
    assertThat(vertex).isNotNull();
    JobVertexID vertexId = vertex.getJobVertexID();
    // get the metric list
    AggregatedMetricsResponseBody metricsResponseBody = getMetricList(endpoint, jobId, vertexId);
    // get the metric query filters
    String queryParam = metricsResponseBody.getMetrics().stream().filter(m -> filterByMetricName(m.getId(), sourceOrSinkName, metricName, filter)).map(m -> m.getId()).collect(Collectors.joining(","));
    if (StringUtils.isNullOrWhitespaceOnly(queryParam)) {
        throw new IllegalStateException(String.format("Cannot find metric[%s] for operator [%s].", metricName, sourceOrSinkName));
    }
    AggregatedMetricsResponseBody metricsResponse = getMetrics(endpoint, jobId, vertexId, queryParam);
    Collection<AggregatedMetric> metrics = metricsResponse.getMetrics();
    if (metrics == null || metrics.isEmpty()) {
        throw new IllegalStateException(String.format("Cannot find metric[%s] for operator [%s] with filter [%s].", metricName, sourceOrSinkName, filter));
    }
    return metrics.iterator().next().getSum();
}