Examples with JobDetails org.apache.flink.runtime.messages.webmonitor.JobDetails used on opensource projects

Example 1 with JobDetails

use of org.apache.flink.runtime.messages.webmonitor.JobDetails in project flink by apache.

the class CurrentJobsOverviewHandler method handleJsonRequest.

@Override
public String handleJsonRequest(Map<String, String> pathParams, Map<String, String> queryParams, ActorGateway jobManager) throws Exception {
    try {
        if (jobManager != null) {
            Future<Object> future = jobManager.ask(new RequestJobDetails(includeRunningJobs, includeFinishedJobs), timeout);
            MultipleJobsDetails result = (MultipleJobsDetails) Await.result(future, timeout);
            final long now = System.currentTimeMillis();
            StringWriter writer = new StringWriter();
            JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer);
            gen.writeStartObject();
            if (includeRunningJobs && includeFinishedJobs) {
                gen.writeArrayFieldStart("running");
                for (JobDetails detail : result.getRunningJobs()) {
                    writeJobDetailOverviewAsJson(detail, gen, now);
                }
                gen.writeEndArray();
                gen.writeArrayFieldStart("finished");
                for (JobDetails detail : result.getFinishedJobs()) {
                    writeJobDetailOverviewAsJson(detail, gen, now);
                }
                gen.writeEndArray();
            } else {
                gen.writeArrayFieldStart("jobs");
                for (JobDetails detail : includeRunningJobs ? result.getRunningJobs() : result.getFinishedJobs()) {
                    writeJobDetailOverviewAsJson(detail, gen, now);
                }
                gen.writeEndArray();
            }
            gen.writeEndObject();
            gen.close();
            return writer.toString();
        } else {
            throw new Exception("No connection to the leading JobManager.");
        }
    } catch (Exception e) {
        throw new Exception("Failed to fetch the status overview: " + e.getMessage(), e);
    }
}

Example 2 with JobDetails

use of org.apache.flink.runtime.messages.webmonitor.JobDetails in project flink by apache.

the class WebMonitorMessagesTest method randomJobDetails.

private JobDetails[] randomJobDetails(Random rnd) {
    final JobDetails[] details = new JobDetails[rnd.nextInt(10)];
    for (int k = 0; k < details.length; k++) {
        int[] numVerticesPerState = new int[ExecutionState.values().length];
        int numTotal = 0;
        for (int i = 0; i < numVerticesPerState.length; i++) {
            int count = rnd.nextInt(55);
            numVerticesPerState[i] = count;
            numTotal += count;
        }
        long time = rnd.nextLong();
        long endTime = rnd.nextBoolean() ? -1L : time + rnd.nextInt();
        long lastModified = endTime == -1 ? time + rnd.nextInt() : endTime;
        String name = new GenericMessageTester.StringInstantiator().instantiate(rnd);
        JobID jid = new JobID();
        JobStatus status = JobStatus.values()[rnd.nextInt(JobStatus.values().length)];
        details[k] = new JobDetails(jid, name, time, endTime, status, lastModified, numVerticesPerState, numTotal);
    }
    return details;
}

Example 3 with JobDetails

use of org.apache.flink.runtime.messages.webmonitor.JobDetails in project flink by apache.

the class CurrentJobsOverviewHandlerTest method testArchiver.

@Test
public void testArchiver() throws Exception {
    JsonArchivist archivist = new CurrentJobsOverviewHandler.CurrentJobsOverviewJsonArchivist();
    AccessExecutionGraph originalJob = ArchivedJobGenerationUtils.getTestJob();
    JobDetails expectedDetails = WebMonitorUtils.createDetailsForJob(originalJob);
    Collection<ArchivedJson> archives = archivist.archiveJsonWithPath(originalJob);
    Assert.assertEquals(1, archives.size());
    ArchivedJson archive = archives.iterator().next();
    Assert.assertEquals("/joboverview", archive.getPath());
    JsonNode result = ArchivedJobGenerationUtils.mapper.readTree(archive.getJson());
    ArrayNode running = (ArrayNode) result.get("running");
    Assert.assertEquals(0, running.size());
    ArrayNode finished = (ArrayNode) result.get("finished");
    Assert.assertEquals(1, finished.size());
    compareJobOverview(expectedDetails, finished.get(0).toString());
}

Example 4 with JobDetails

use of org.apache.flink.runtime.messages.webmonitor.JobDetails in project flink by apache.

the class CurrentJobsOverviewHandlerTest method testJsonGeneration.

@Test
public void testJsonGeneration() throws Exception {
    AccessExecutionGraph originalJob = ArchivedJobGenerationUtils.getTestJob();
    JobDetails expectedDetails = WebMonitorUtils.createDetailsForJob(originalJob);
    StringWriter writer = new StringWriter();
    try (JsonGenerator gen = ArchivedJobGenerationUtils.jacksonFactory.createGenerator(writer)) {
        CurrentJobsOverviewHandler.writeJobDetailOverviewAsJson(expectedDetails, gen, 0);
    }
    compareJobOverview(expectedDetails, writer.toString());
}

Example 5 with JobDetails

use of org.apache.flink.runtime.messages.webmonitor.JobDetails in project flink by apache.

the class MetricFetcher method fetchMetrics.

private void fetchMetrics() {
    try {
        Option<scala.Tuple2<ActorGateway, Integer>> jobManagerGatewayAndWebPort = retriever.getJobManagerGatewayAndWebPort();
        if (jobManagerGatewayAndWebPort.isDefined()) {
            ActorGateway jobManager = jobManagerGatewayAndWebPort.get()._1();
            /**
				 * Remove all metrics that belong to a job that is not running and no longer archived.
				 */
            Future<Object> jobDetailsFuture = jobManager.ask(new RequestJobDetails(true, true), timeout);
            jobDetailsFuture.onSuccess(new OnSuccess<Object>() {

                @Override
                public void onSuccess(Object result) throws Throwable {
                    MultipleJobsDetails details = (MultipleJobsDetails) result;
                    ArrayList<String> toRetain = new ArrayList<>();
                    for (JobDetails job : details.getRunningJobs()) {
                        toRetain.add(job.getJobId().toString());
                    }
                    for (JobDetails job : details.getFinishedJobs()) {
                        toRetain.add(job.getJobId().toString());
                    }
                    synchronized (metrics) {
                        metrics.jobs.keySet().retainAll(toRetain);
                    }
                }
            }, ctx);
            logErrorOnFailure(jobDetailsFuture, "Fetching of JobDetails failed.");
            String jobManagerPath = jobManager.path();
            String queryServicePath = jobManagerPath.substring(0, jobManagerPath.lastIndexOf('/') + 1) + MetricQueryService.METRIC_QUERY_SERVICE_NAME;
            ActorRef jobManagerQueryService = actorSystem.actorFor(queryServicePath);
            queryMetrics(jobManagerQueryService);
            /**
				 * We first request the list of all registered task managers from the job manager, and then
				 * request the respective metric dump from each task manager.
				 *
				 * All stored metrics that do not belong to a registered task manager will be removed.
				 */
            Future<Object> registeredTaskManagersFuture = jobManager.ask(JobManagerMessages.getRequestRegisteredTaskManagers(), timeout);
            registeredTaskManagersFuture.onSuccess(new OnSuccess<Object>() {

                @Override
                public void onSuccess(Object result) throws Throwable {
                    Iterable<Instance> taskManagers = ((JobManagerMessages.RegisteredTaskManagers) result).asJavaIterable();
                    List<String> activeTaskManagers = new ArrayList<>();
                    for (Instance taskManager : taskManagers) {
                        activeTaskManagers.add(taskManager.getId().toString());
                        String taskManagerPath = taskManager.getTaskManagerGateway().getAddress();
                        String queryServicePath = taskManagerPath.substring(0, taskManagerPath.lastIndexOf('/') + 1) + MetricQueryService.METRIC_QUERY_SERVICE_NAME + "_" + taskManager.getTaskManagerID().getResourceIdString();
                        ActorRef taskManagerQueryService = actorSystem.actorFor(queryServicePath);
                        queryMetrics(taskManagerQueryService);
                    }
                    synchronized (metrics) {
                        // remove all metrics belonging to unregistered task managers
                        metrics.taskManagers.keySet().retainAll(activeTaskManagers);
                    }
                }
            }, ctx);
            logErrorOnFailure(registeredTaskManagersFuture, "Fetchin list of registered TaskManagers failed.");
        }
    } catch (Exception e) {
        LOG.warn("Exception while fetching metrics.", e);
    }
}