Examples with AccessExecutionVertex org.apache.flink.runtime.executiongraph.AccessExecutionVertex used on opensource projects

Example 11 with AccessExecutionVertex

use of org.apache.flink.runtime.executiongraph.AccessExecutionVertex in project flink by apache.

the class JobVertexTaskManagersHandler method createVertexDetailsByTaskManagerJson.

public static String createVertexDetailsByTaskManagerJson(AccessExecutionJobVertex jobVertex, String jobID, @Nullable MetricFetcher fetcher) throws IOException {
    StringWriter writer = new StringWriter();
    JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer);
    // Build a map that groups tasks by TaskManager
    Map<String, List<AccessExecutionVertex>> taskManagerVertices = new HashMap<>();
    for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) {
        TaskManagerLocation location = vertex.getCurrentAssignedResourceLocation();
        String taskManager = location == null ? "(unassigned)" : location.getHostname() + ":" + location.dataPort();
        List<AccessExecutionVertex> vertices = taskManagerVertices.get(taskManager);
        if (vertices == null) {
            vertices = new ArrayList<>();
            taskManagerVertices.put(taskManager, vertices);
        }
        vertices.add(vertex);
    }
    // Build JSON response
    final long now = System.currentTimeMillis();
    gen.writeStartObject();
    gen.writeStringField("id", jobVertex.getJobVertexId().toString());
    gen.writeStringField("name", jobVertex.getName());
    gen.writeNumberField("now", now);
    gen.writeArrayFieldStart("taskmanagers");
    for (Map.Entry<String, List<AccessExecutionVertex>> entry : taskManagerVertices.entrySet()) {
        String host = entry.getKey();
        List<AccessExecutionVertex> taskVertices = entry.getValue();
        int[] tasksPerState = new int[ExecutionState.values().length];
        long startTime = Long.MAX_VALUE;
        long endTime = 0;
        boolean allFinished = true;
        MutableIOMetrics counts = new MutableIOMetrics();
        for (AccessExecutionVertex vertex : taskVertices) {
            final ExecutionState state = vertex.getExecutionState();
            tasksPerState[state.ordinal()]++;
            // take the earliest start time
            long started = vertex.getStateTimestamp(ExecutionState.DEPLOYING);
            if (started > 0) {
                startTime = Math.min(startTime, started);
            }
            allFinished &= state.isTerminal();
            endTime = Math.max(endTime, vertex.getStateTimestamp(state));
            counts.addIOMetrics(vertex.getCurrentExecutionAttempt(), fetcher, jobID, jobVertex.getJobVertexId().toString());
        }
        long duration;
        if (startTime < Long.MAX_VALUE) {
            if (allFinished) {
                duration = endTime - startTime;
            } else {
                endTime = -1L;
                duration = now - startTime;
            }
        } else {
            startTime = -1L;
            endTime = -1L;
            duration = -1L;
        }
        ExecutionState jobVertexState = ExecutionJobVertex.getAggregateJobVertexState(tasksPerState, taskVertices.size());
        gen.writeStartObject();
        gen.writeStringField("host", host);
        gen.writeStringField("status", jobVertexState.name());
        gen.writeNumberField("start-time", startTime);
        gen.writeNumberField("end-time", endTime);
        gen.writeNumberField("duration", duration);
        counts.writeIOMetricsAsJson(gen);
        gen.writeObjectFieldStart("status-counts");
        for (ExecutionState state : ExecutionState.values()) {
            gen.writeNumberField(state.name(), tasksPerState[state.ordinal()]);
        }
        gen.writeEndObject();
        gen.writeEndObject();
    }
    gen.writeEndArray();
    gen.writeEndObject();
    gen.close();
    return writer.toString();
}

Example 12 with AccessExecutionVertex

use of org.apache.flink.runtime.executiongraph.AccessExecutionVertex in project flink by apache.

the class SubtasksAllAccumulatorsHandler method createSubtasksAccumulatorsJson.

public static String createSubtasksAccumulatorsJson(AccessExecutionJobVertex jobVertex) throws IOException {
    StringWriter writer = new StringWriter();
    JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer);
    gen.writeStartObject();
    gen.writeStringField("id", jobVertex.getJobVertexId().toString());
    gen.writeNumberField("parallelism", jobVertex.getParallelism());
    gen.writeArrayFieldStart("subtasks");
    int num = 0;
    for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) {
        TaskManagerLocation location = vertex.getCurrentAssignedResourceLocation();
        String locationString = location == null ? "(unassigned)" : location.getHostname();
        gen.writeStartObject();
        gen.writeNumberField("subtask", num++);
        gen.writeNumberField("attempt", vertex.getCurrentExecutionAttempt().getAttemptNumber());
        gen.writeStringField("host", locationString);
        StringifiedAccumulatorResult[] accs = vertex.getCurrentExecutionAttempt().getUserAccumulatorsStringified();
        gen.writeArrayFieldStart("user-accumulators");
        for (StringifiedAccumulatorResult acc : accs) {
            gen.writeStartObject();
            gen.writeStringField("name", acc.getName());
            gen.writeStringField("type", acc.getType());
            gen.writeStringField("value", acc.getValue());
            gen.writeEndObject();
        }
        gen.writeEndArray();
        gen.writeEndObject();
    }
    gen.writeEndArray();
    gen.writeEndObject();
    gen.close();
    return writer.toString();
}

Example 13 with AccessExecutionVertex

use of org.apache.flink.runtime.executiongraph.AccessExecutionVertex in project flink by apache.

the class JobVertexDetailsHandlerTest method compareVertexDetails.

private static void compareVertexDetails(AccessExecutionJobVertex originalTask, String json) throws IOException {
    JsonNode result = ArchivedJobGenerationUtils.mapper.readTree(json);
    Assert.assertEquals(originalTask.getJobVertexId().toString(), result.get("id").asText());
    Assert.assertEquals(originalTask.getName(), result.get("name").asText());
    Assert.assertEquals(originalTask.getParallelism(), result.get("parallelism").asInt());
    Assert.assertTrue(result.get("now").asLong() > 0);
    ArrayNode subtasks = (ArrayNode) result.get("subtasks");
    Assert.assertEquals(originalTask.getTaskVertices().length, subtasks.size());
    for (int x = 0; x < originalTask.getTaskVertices().length; x++) {
        AccessExecutionVertex expectedSubtask = originalTask.getTaskVertices()[x];
        JsonNode subtask = subtasks.get(x);
        Assert.assertEquals(x, subtask.get("subtask").asInt());
        Assert.assertEquals(expectedSubtask.getExecutionState().name(), subtask.get("status").asText());
        Assert.assertEquals(expectedSubtask.getCurrentExecutionAttempt().getAttemptNumber(), subtask.get("attempt").asInt());
        TaskManagerLocation location = expectedSubtask.getCurrentAssignedResourceLocation();
        String expectedLocationString = location.getHostname() + ":" + location.dataPort();
        Assert.assertEquals(expectedLocationString, subtask.get("host").asText());
        long start = expectedSubtask.getStateTimestamp(ExecutionState.DEPLOYING);
        Assert.assertEquals(start, subtask.get("start-time").asLong());
        long end = expectedSubtask.getStateTimestamp(ExecutionState.FINISHED);
        Assert.assertEquals(end, subtask.get("end-time").asLong());
        Assert.assertEquals(end - start, subtask.get("duration").asLong());
        ArchivedJobGenerationUtils.compareIoMetrics(expectedSubtask.getCurrentExecutionAttempt().getIOMetrics(), subtask.get("metrics"));
    }
}

Example 14 with AccessExecutionVertex

use of org.apache.flink.runtime.executiongraph.AccessExecutionVertex in project flink by apache.

the class JobVertexTaskManagersHandlerTest method testArchiver.

@Test
public void testArchiver() throws Exception {
    JsonArchivist archivist = new JobVertexTaskManagersHandler.JobVertexTaskManagersJsonArchivist();
    AccessExecutionGraph originalJob = ArchivedJobGenerationUtils.getTestJob();
    AccessExecutionJobVertex originalTask = ArchivedJobGenerationUtils.getTestTask();
    AccessExecutionVertex originalSubtask = ArchivedJobGenerationUtils.getTestSubtask();
    Collection<ArchivedJson> archives = archivist.archiveJsonWithPath(originalJob);
    Assert.assertEquals(1, archives.size());
    ArchivedJson archive = archives.iterator().next();
    Assert.assertEquals("/jobs/" + originalJob.getJobID() + "/vertices/" + originalTask.getJobVertexId() + "/taskmanagers", archive.getPath());
    compareVertexTaskManagers(originalTask, originalSubtask, archive.getJson());
}

Example 15 with AccessExecutionVertex

use of org.apache.flink.runtime.executiongraph.AccessExecutionVertex in project flink by apache.

the class WebMonitorUtils method createDetailsForJob.

public static JobDetails createDetailsForJob(AccessExecutionGraph job) {
    JobStatus status = job.getState();
    long started = job.getStatusTimestamp(JobStatus.CREATED);
    long finished = status.isGloballyTerminalState() ? job.getStatusTimestamp(status) : -1L;
    int[] countsPerStatus = new int[ExecutionState.values().length];
    long lastChanged = 0;
    int numTotalTasks = 0;
    for (AccessExecutionJobVertex ejv : job.getVerticesTopologically()) {
        AccessExecutionVertex[] vertices = ejv.getTaskVertices();
        numTotalTasks += vertices.length;
        for (AccessExecutionVertex vertex : vertices) {
            ExecutionState state = vertex.getExecutionState();
            countsPerStatus[state.ordinal()]++;
            lastChanged = Math.max(lastChanged, vertex.getStateTimestamp(state));
        }
    }
    lastChanged = Math.max(lastChanged, finished);
    return new JobDetails(job.getJobID(), job.getJobName(), started, finished, status, lastChanged, countsPerStatus, numTotalTasks);
}