Examples with AccessExecutionJobVertex org.apache.flink.runtime.executiongraph.AccessExecutionJobVertex used on opensource projects

Example 11 with AccessExecutionJobVertex

use of org.apache.flink.runtime.executiongraph.AccessExecutionJobVertex in project flink by apache.

the class JobDetailsHandler method createJobDetailsJson.

public static String createJobDetailsJson(AccessExecutionGraph graph, @Nullable MetricFetcher fetcher) throws IOException {
    final StringWriter writer = new StringWriter();
    final JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer);
    final long now = System.currentTimeMillis();
    gen.writeStartObject();
    // basic info
    gen.writeStringField("jid", graph.getJobID().toString());
    gen.writeStringField("name", graph.getJobName());
    gen.writeBooleanField("isStoppable", graph.isStoppable());
    gen.writeStringField("state", graph.getState().name());
    // times and duration
    final long jobStartTime = graph.getStatusTimestamp(JobStatus.CREATED);
    final long jobEndTime = graph.getState().isGloballyTerminalState() ? graph.getStatusTimestamp(graph.getState()) : -1L;
    gen.writeNumberField("start-time", jobStartTime);
    gen.writeNumberField("end-time", jobEndTime);
    gen.writeNumberField("duration", (jobEndTime > 0 ? jobEndTime : now) - jobStartTime);
    gen.writeNumberField("now", now);
    // timestamps
    gen.writeObjectFieldStart("timestamps");
    for (JobStatus status : JobStatus.values()) {
        gen.writeNumberField(status.name(), graph.getStatusTimestamp(status));
    }
    gen.writeEndObject();
    // job vertices
    int[] jobVerticesPerState = new int[ExecutionState.values().length];
    gen.writeArrayFieldStart("vertices");
    for (AccessExecutionJobVertex ejv : graph.getVerticesTopologically()) {
        int[] tasksPerState = new int[ExecutionState.values().length];
        long startTime = Long.MAX_VALUE;
        long endTime = 0;
        boolean allFinished = true;
        for (AccessExecutionVertex vertex : ejv.getTaskVertices()) {
            final ExecutionState state = vertex.getExecutionState();
            tasksPerState[state.ordinal()]++;
            // take the earliest start time
            long started = vertex.getStateTimestamp(ExecutionState.DEPLOYING);
            if (started > 0) {
                startTime = Math.min(startTime, started);
            }
            allFinished &= state.isTerminal();
            endTime = Math.max(endTime, vertex.getStateTimestamp(state));
        }
        long duration;
        if (startTime < Long.MAX_VALUE) {
            if (allFinished) {
                duration = endTime - startTime;
            } else {
                endTime = -1L;
                duration = now - startTime;
            }
        } else {
            startTime = -1L;
            endTime = -1L;
            duration = -1L;
        }
        ExecutionState jobVertexState = ExecutionJobVertex.getAggregateJobVertexState(tasksPerState, ejv.getParallelism());
        jobVerticesPerState[jobVertexState.ordinal()]++;
        gen.writeStartObject();
        gen.writeStringField("id", ejv.getJobVertexId().toString());
        gen.writeStringField("name", ejv.getName());
        gen.writeNumberField("parallelism", ejv.getParallelism());
        gen.writeStringField("status", jobVertexState.name());
        gen.writeNumberField("start-time", startTime);
        gen.writeNumberField("end-time", endTime);
        gen.writeNumberField("duration", duration);
        gen.writeObjectFieldStart("tasks");
        for (ExecutionState state : ExecutionState.values()) {
            gen.writeNumberField(state.name(), tasksPerState[state.ordinal()]);
        }
        gen.writeEndObject();
        MutableIOMetrics counts = new MutableIOMetrics();
        for (AccessExecutionVertex vertex : ejv.getTaskVertices()) {
            counts.addIOMetrics(vertex.getCurrentExecutionAttempt(), fetcher, graph.getJobID().toString(), ejv.getJobVertexId().toString());
        }
        counts.writeIOMetricsAsJson(gen);
        gen.writeEndObject();
    }
    gen.writeEndArray();
    gen.writeObjectFieldStart("status-counts");
    for (ExecutionState state : ExecutionState.values()) {
        gen.writeNumberField(state.name(), jobVerticesPerState[state.ordinal()]);
    }
    gen.writeEndObject();
    gen.writeFieldName("plan");
    gen.writeRawValue(graph.getJsonPlan());
    gen.writeEndObject();
    gen.close();
    return writer.toString();
}

Example 12 with AccessExecutionJobVertex

use of org.apache.flink.runtime.executiongraph.AccessExecutionJobVertex in project flink by apache.

the class SubtasksTimesHandlerTest method testArchiver.

@Test
public void testArchiver() throws Exception {
    JsonArchivist archivist = new SubtasksTimesHandler.SubtasksTimesJsonArchivist();
    AccessExecutionGraph originalJob = ArchivedJobGenerationUtils.getTestJob();
    AccessExecutionJobVertex originalTask = ArchivedJobGenerationUtils.getTestTask();
    AccessExecution originalAttempt = ArchivedJobGenerationUtils.getTestAttempt();
    Collection<ArchivedJson> archives = archivist.archiveJsonWithPath(originalJob);
    Assert.assertEquals(1, archives.size());
    ArchivedJson archive = archives.iterator().next();
    Assert.assertEquals("/jobs/" + originalJob.getJobID() + "/vertices/" + originalTask.getJobVertexId() + "/subtasktimes", archive.getPath());
    compareSubtaskTimes(originalTask, originalAttempt, archive.getJson());
}

Example 13 with AccessExecutionJobVertex

use of org.apache.flink.runtime.executiongraph.AccessExecutionJobVertex in project flink by apache.

the class JobVertexAccumulatorsHandlerTest method testJsonGeneration.

@Test
public void testJsonGeneration() throws Exception {
    AccessExecutionJobVertex originalTask = ArchivedJobGenerationUtils.getTestTask();
    String json = JobVertexAccumulatorsHandler.createVertexAccumulatorsJson(originalTask);
    compareAccumulators(originalTask, json);
}

Example 14 with AccessExecutionJobVertex

use of org.apache.flink.runtime.executiongraph.AccessExecutionJobVertex in project flink by apache.

the class JobVertexDetailsHandlerTest method testArchiver.

@Test
public void testArchiver() throws Exception {
    JsonArchivist archivist = new JobVertexDetailsHandler.JobVertexDetailsJsonArchivist();
    AccessExecutionGraph originalJob = ArchivedJobGenerationUtils.getTestJob();
    AccessExecutionJobVertex originalTask = ArchivedJobGenerationUtils.getTestTask();
    Collection<ArchivedJson> archives = archivist.archiveJsonWithPath(originalJob);
    Assert.assertEquals(1, archives.size());
    ArchivedJson archive = archives.iterator().next();
    Assert.assertEquals("/jobs/" + originalJob.getJobID() + "/vertices/" + originalTask.getJobVertexId(), archive.getPath());
    compareVertexDetails(originalTask, archive.getJson());
}

Example 15 with AccessExecutionJobVertex

use of org.apache.flink.runtime.executiongraph.AccessExecutionJobVertex in project flink by apache.

the class JobVertexTaskManagersHandlerTest method testArchiver.

@Test
public void testArchiver() throws Exception {
    JsonArchivist archivist = new JobVertexTaskManagersHandler.JobVertexTaskManagersJsonArchivist();
    AccessExecutionGraph originalJob = ArchivedJobGenerationUtils.getTestJob();
    AccessExecutionJobVertex originalTask = ArchivedJobGenerationUtils.getTestTask();
    AccessExecutionVertex originalSubtask = ArchivedJobGenerationUtils.getTestSubtask();
    Collection<ArchivedJson> archives = archivist.archiveJsonWithPath(originalJob);
    Assert.assertEquals(1, archives.size());
    ArchivedJson archive = archives.iterator().next();
    Assert.assertEquals("/jobs/" + originalJob.getJobID() + "/vertices/" + originalTask.getJobVertexId() + "/taskmanagers", archive.getPath());
    compareVertexTaskManagers(originalTask, originalSubtask, archive.getJson());
}