Examples with AccessExecutionVertex org.apache.flink.runtime.executiongraph.AccessExecutionVertex used on opensource projects

Example 6 with AccessExecutionVertex

use of org.apache.flink.runtime.executiongraph.AccessExecutionVertex in project flink by apache.

the class SubtasksAllAccumulatorsHandlerTest method compareSubtaskAccumulators.

private static void compareSubtaskAccumulators(AccessExecutionJobVertex originalTask, String json) throws IOException {
    JsonNode result = ArchivedJobGenerationUtils.mapper.readTree(json);
    Assert.assertEquals(originalTask.getJobVertexId().toString(), result.get("id").asText());
    Assert.assertEquals(originalTask.getParallelism(), result.get("parallelism").asInt());
    ArrayNode subtasks = (ArrayNode) result.get("subtasks");
    Assert.assertEquals(originalTask.getTaskVertices().length, subtasks.size());
    for (int x = 0; x < originalTask.getTaskVertices().length; x++) {
        JsonNode subtask = subtasks.get(x);
        AccessExecutionVertex expectedSubtask = originalTask.getTaskVertices()[x];
        Assert.assertEquals(x, subtask.get("subtask").asInt());
        Assert.assertEquals(expectedSubtask.getCurrentExecutionAttempt().getAttemptNumber(), subtask.get("attempt").asInt());
        Assert.assertEquals(expectedSubtask.getCurrentAssignedResourceLocation().getHostname(), subtask.get("host").asText());
        ArchivedJobGenerationUtils.compareStringifiedAccumulators(expectedSubtask.getCurrentExecutionAttempt().getUserAccumulatorsStringified(), (ArrayNode) subtask.get("user-accumulators"));
    }
}

Example 7 with AccessExecutionVertex

use of org.apache.flink.runtime.executiongraph.AccessExecutionVertex in project flink by apache.

the class AbstractSubtaskRequestHandler method handleRequest.

@Override
public final String handleRequest(AccessExecutionJobVertex jobVertex, Map<String, String> params) throws Exception {
    final String subtaskNumberString = params.get("subtasknum");
    if (subtaskNumberString == null) {
        throw new RuntimeException("Subtask number parameter missing");
    }
    final int subtask;
    try {
        subtask = Integer.parseInt(subtaskNumberString);
    } catch (NumberFormatException e) {
        throw new RuntimeException("Invalid subtask number parameter");
    }
    if (subtask < 0 || subtask >= jobVertex.getParallelism()) {
        throw new RuntimeException("subtask does not exist: " + subtask);
    }
    final AccessExecutionVertex vertex = jobVertex.getTaskVertices()[subtask];
    return handleRequest(vertex, params);
}

Example 8 with AccessExecutionVertex

use of org.apache.flink.runtime.executiongraph.AccessExecutionVertex in project flink by apache.

the class JobDetailsHandler method createJobDetailsJson.

public static String createJobDetailsJson(AccessExecutionGraph graph, @Nullable MetricFetcher fetcher) throws IOException {
    final StringWriter writer = new StringWriter();
    final JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer);
    final long now = System.currentTimeMillis();
    gen.writeStartObject();
    // basic info
    gen.writeStringField("jid", graph.getJobID().toString());
    gen.writeStringField("name", graph.getJobName());
    gen.writeBooleanField("isStoppable", graph.isStoppable());
    gen.writeStringField("state", graph.getState().name());
    // times and duration
    final long jobStartTime = graph.getStatusTimestamp(JobStatus.CREATED);
    final long jobEndTime = graph.getState().isGloballyTerminalState() ? graph.getStatusTimestamp(graph.getState()) : -1L;
    gen.writeNumberField("start-time", jobStartTime);
    gen.writeNumberField("end-time", jobEndTime);
    gen.writeNumberField("duration", (jobEndTime > 0 ? jobEndTime : now) - jobStartTime);
    gen.writeNumberField("now", now);
    // timestamps
    gen.writeObjectFieldStart("timestamps");
    for (JobStatus status : JobStatus.values()) {
        gen.writeNumberField(status.name(), graph.getStatusTimestamp(status));
    }
    gen.writeEndObject();
    // job vertices
    int[] jobVerticesPerState = new int[ExecutionState.values().length];
    gen.writeArrayFieldStart("vertices");
    for (AccessExecutionJobVertex ejv : graph.getVerticesTopologically()) {
        int[] tasksPerState = new int[ExecutionState.values().length];
        long startTime = Long.MAX_VALUE;
        long endTime = 0;
        boolean allFinished = true;
        for (AccessExecutionVertex vertex : ejv.getTaskVertices()) {
            final ExecutionState state = vertex.getExecutionState();
            tasksPerState[state.ordinal()]++;
            // take the earliest start time
            long started = vertex.getStateTimestamp(ExecutionState.DEPLOYING);
            if (started > 0) {
                startTime = Math.min(startTime, started);
            }
            allFinished &= state.isTerminal();
            endTime = Math.max(endTime, vertex.getStateTimestamp(state));
        }
        long duration;
        if (startTime < Long.MAX_VALUE) {
            if (allFinished) {
                duration = endTime - startTime;
            } else {
                endTime = -1L;
                duration = now - startTime;
            }
        } else {
            startTime = -1L;
            endTime = -1L;
            duration = -1L;
        }
        ExecutionState jobVertexState = ExecutionJobVertex.getAggregateJobVertexState(tasksPerState, ejv.getParallelism());
        jobVerticesPerState[jobVertexState.ordinal()]++;
        gen.writeStartObject();
        gen.writeStringField("id", ejv.getJobVertexId().toString());
        gen.writeStringField("name", ejv.getName());
        gen.writeNumberField("parallelism", ejv.getParallelism());
        gen.writeStringField("status", jobVertexState.name());
        gen.writeNumberField("start-time", startTime);
        gen.writeNumberField("end-time", endTime);
        gen.writeNumberField("duration", duration);
        gen.writeObjectFieldStart("tasks");
        for (ExecutionState state : ExecutionState.values()) {
            gen.writeNumberField(state.name(), tasksPerState[state.ordinal()]);
        }
        gen.writeEndObject();
        MutableIOMetrics counts = new MutableIOMetrics();
        for (AccessExecutionVertex vertex : ejv.getTaskVertices()) {
            counts.addIOMetrics(vertex.getCurrentExecutionAttempt(), fetcher, graph.getJobID().toString(), ejv.getJobVertexId().toString());
        }
        counts.writeIOMetricsAsJson(gen);
        gen.writeEndObject();
    }
    gen.writeEndArray();
    gen.writeObjectFieldStart("status-counts");
    for (ExecutionState state : ExecutionState.values()) {
        gen.writeNumberField(state.name(), jobVerticesPerState[state.ordinal()]);
    }
    gen.writeEndObject();
    gen.writeFieldName("plan");
    gen.writeRawValue(graph.getJsonPlan());
    gen.writeEndObject();
    gen.close();
    return writer.toString();
}

Example 9 with AccessExecutionVertex

use of org.apache.flink.runtime.executiongraph.AccessExecutionVertex in project flink by apache.

the class JobExceptionsHandler method createJobExceptionsJson.

public static String createJobExceptionsJson(AccessExecutionGraph graph) throws IOException {
    StringWriter writer = new StringWriter();
    JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer);
    gen.writeStartObject();
    // most important is the root failure cause
    String rootException = graph.getFailureCauseAsString();
    if (rootException != null && !rootException.equals(ExceptionUtils.STRINGIFIED_NULL_EXCEPTION)) {
        gen.writeStringField("root-exception", rootException);
    }
    // we additionally collect all exceptions (up to a limit) that occurred in the individual tasks
    gen.writeArrayFieldStart("all-exceptions");
    int numExceptionsSoFar = 0;
    boolean truncated = false;
    for (AccessExecutionVertex task : graph.getAllExecutionVertices()) {
        String t = task.getFailureCauseAsString();
        if (t != null && !t.equals(ExceptionUtils.STRINGIFIED_NULL_EXCEPTION)) {
            if (numExceptionsSoFar >= MAX_NUMBER_EXCEPTION_TO_REPORT) {
                truncated = true;
                break;
            }
            TaskManagerLocation location = task.getCurrentAssignedResourceLocation();
            String locationString = location != null ? location.getFQDNHostname() + ':' + location.dataPort() : "(unassigned)";
            gen.writeStartObject();
            gen.writeStringField("exception", t);
            gen.writeStringField("task", task.getTaskNameWithSubtaskIndex());
            gen.writeStringField("location", locationString);
            gen.writeEndObject();
            numExceptionsSoFar++;
        }
    }
    gen.writeEndArray();
    gen.writeBooleanField("truncated", truncated);
    gen.writeEndObject();
    gen.close();
    return writer.toString();
}

Example 10 with AccessExecutionVertex

use of org.apache.flink.runtime.executiongraph.AccessExecutionVertex in project flink by apache.

the class JobVertexDetailsHandler method createVertexDetailsJson.

public static String createVertexDetailsJson(AccessExecutionJobVertex jobVertex, String jobID, @Nullable MetricFetcher fetcher) throws IOException {
    final long now = System.currentTimeMillis();
    StringWriter writer = new StringWriter();
    JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer);
    gen.writeStartObject();
    gen.writeStringField("id", jobVertex.getJobVertexId().toString());
    gen.writeStringField("name", jobVertex.getName());
    gen.writeNumberField("parallelism", jobVertex.getParallelism());
    gen.writeNumberField("now", now);
    gen.writeArrayFieldStart("subtasks");
    int num = 0;
    for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) {
        final ExecutionState status = vertex.getExecutionState();
        TaskManagerLocation location = vertex.getCurrentAssignedResourceLocation();
        String locationString = location == null ? "(unassigned)" : location.getHostname() + ":" + location.dataPort();
        long startTime = vertex.getStateTimestamp(ExecutionState.DEPLOYING);
        if (startTime == 0) {
            startTime = -1;
        }
        long endTime = status.isTerminal() ? vertex.getStateTimestamp(status) : -1;
        long duration = startTime > 0 ? ((endTime > 0 ? endTime : now) - startTime) : -1;
        gen.writeStartObject();
        gen.writeNumberField("subtask", num);
        gen.writeStringField("status", status.name());
        gen.writeNumberField("attempt", vertex.getCurrentExecutionAttempt().getAttemptNumber());
        gen.writeStringField("host", locationString);
        gen.writeNumberField("start-time", startTime);
        gen.writeNumberField("end-time", endTime);
        gen.writeNumberField("duration", duration);
        MutableIOMetrics counts = new MutableIOMetrics();
        counts.addIOMetrics(vertex.getCurrentExecutionAttempt(), fetcher, jobID, jobVertex.getJobVertexId().toString());
        counts.writeIOMetricsAsJson(gen);
        gen.writeEndObject();
        num++;
    }
    gen.writeEndArray();
    gen.writeEndObject();
    gen.close();
    return writer.toString();
}