Examples with TaskManagerLocation org.apache.flink.runtime.taskmanager.TaskManagerLocation used on opensource projects

Example 81 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class JobExceptionsHandler method createJobExceptionsInfo.

private static JobExceptionsInfoWithHistory createJobExceptionsInfo(ExecutionGraphInfo executionGraphInfo, int exceptionToReportMaxSize) {
    final ArchivedExecutionGraph executionGraph = executionGraphInfo.getArchivedExecutionGraph();
    if (executionGraph.getFailureInfo() == null) {
        return new JobExceptionsInfoWithHistory();
    }
    List<JobExceptionsInfo.ExecutionExceptionInfo> taskExceptionList = new ArrayList<>();
    boolean truncated = false;
    for (AccessExecutionVertex task : executionGraph.getAllExecutionVertices()) {
        Optional<ErrorInfo> failure = task.getFailureInfo();
        if (failure.isPresent()) {
            if (taskExceptionList.size() >= exceptionToReportMaxSize) {
                truncated = true;
                break;
            }
            TaskManagerLocation location = task.getCurrentAssignedResourceLocation();
            String locationString = toString(location);
            long timestamp = task.getStateTimestamp(ExecutionState.FAILED);
            taskExceptionList.add(new JobExceptionsInfo.ExecutionExceptionInfo(failure.get().getExceptionAsString(), task.getTaskNameWithSubtaskIndex(), locationString, timestamp == 0 ? -1 : timestamp));
        }
    }
    final ErrorInfo rootCause = executionGraph.getFailureInfo();
    return new JobExceptionsInfoWithHistory(rootCause.getExceptionAsString(), rootCause.getTimestamp(), taskExceptionList, truncated, createJobExceptionHistory(executionGraphInfo.getExceptionHistory(), exceptionToReportMaxSize));
}

Example 82 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class SubtasksAllAccumulatorsHandler method handleRequest.

@Override
protected SubtasksAllAccumulatorsInfo handleRequest(HandlerRequest<EmptyRequestBody> request, AccessExecutionJobVertex jobVertex) throws RestHandlerException {
    JobVertexID jobVertexId = jobVertex.getJobVertexId();
    int parallelism = jobVertex.getParallelism();
    final List<SubtasksAllAccumulatorsInfo.SubtaskAccumulatorsInfo> subtaskAccumulatorsInfos = new ArrayList<>();
    for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) {
        TaskManagerLocation location = vertex.getCurrentAssignedResourceLocation();
        String locationString = location == null ? "(unassigned)" : location.getHostname();
        StringifiedAccumulatorResult[] accs = vertex.getCurrentExecutionAttempt().getUserAccumulatorsStringified();
        List<UserAccumulator> userAccumulators = new ArrayList<>(accs.length);
        for (StringifiedAccumulatorResult acc : accs) {
            userAccumulators.add(new UserAccumulator(acc.getName(), acc.getType(), acc.getValue()));
        }
        subtaskAccumulatorsInfos.add(new SubtasksAllAccumulatorsInfo.SubtaskAccumulatorsInfo(vertex.getCurrentExecutionAttempt().getParallelSubtaskIndex(), vertex.getCurrentExecutionAttempt().getAttemptNumber(), locationString, userAccumulators));
    }
    return new SubtasksAllAccumulatorsInfo(jobVertexId, parallelism, subtaskAccumulatorsInfos);
}

Example 83 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class JobMasterTest method testJobMasterDisconnectsOldTaskExecutorIfNewSessionIsSeen.

@Test
public void testJobMasterDisconnectsOldTaskExecutorIfNewSessionIsSeen() throws Exception {
    final JobMaster jobMaster = new JobMasterBuilder(jobGraph, rpcService).createJobMaster();
    final CompletableFuture<Void> firstTaskExecutorDisconnectedFuture = new CompletableFuture<>();
    final TestingTaskExecutorGateway firstTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setAddress("firstTaskExecutor").setDisconnectJobManagerConsumer((jobID, throwable) -> firstTaskExecutorDisconnectedFuture.complete(null)).createTestingTaskExecutorGateway();
    final TestingTaskExecutorGateway secondTaskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setAddress("secondTaskExecutor").createTestingTaskExecutorGateway();
    rpcService.registerGateway(firstTaskExecutorGateway.getAddress(), firstTaskExecutorGateway);
    rpcService.registerGateway(secondTaskExecutorGateway.getAddress(), secondTaskExecutorGateway);
    try {
        jobMaster.start();
        final LocalUnresolvedTaskManagerLocation taskManagerLocation = new LocalUnresolvedTaskManagerLocation();
        final UUID firstTaskManagerSessionId = UUID.randomUUID();
        final CompletableFuture<RegistrationResponse> firstRegistrationResponse = jobMaster.registerTaskManager(jobGraph.getJobID(), TaskManagerRegistrationInformation.create(firstTaskExecutorGateway.getAddress(), taskManagerLocation, firstTaskManagerSessionId), testingTimeout);
        assertThat(firstRegistrationResponse.get(), instanceOf(JMTMRegistrationSuccess.class));
        final UUID secondTaskManagerSessionId = UUID.randomUUID();
        final CompletableFuture<RegistrationResponse> secondRegistrationResponse = jobMaster.registerTaskManager(jobGraph.getJobID(), TaskManagerRegistrationInformation.create(secondTaskExecutorGateway.getAddress(), taskManagerLocation, secondTaskManagerSessionId), testingTimeout);
        assertThat(secondRegistrationResponse.get(), instanceOf(JMTMRegistrationSuccess.class));
        // the first TaskExecutor should be disconnected
        firstTaskExecutorDisconnectedFuture.get();
    } finally {
        RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
    }
}

Example 84 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class SubtaskExecutionAttemptDetailsInfo method create.

public static SubtaskExecutionAttemptDetailsInfo create(AccessExecution execution, @Nullable MetricFetcher metricFetcher, JobID jobID, JobVertexID jobVertexID) {
    final ExecutionState status = execution.getState();
    final long now = System.currentTimeMillis();
    final TaskManagerLocation location = execution.getAssignedResourceLocation();
    final String locationString = location == null ? "(unassigned)" : location.getHostname();
    String taskmanagerId = location == null ? "(unassigned)" : location.getResourceID().toString();
    long startTime = execution.getStateTimestamp(ExecutionState.DEPLOYING);
    if (startTime == 0) {
        startTime = -1;
    }
    final long endTime = status.isTerminal() ? execution.getStateTimestamp(status) : -1;
    final long duration = startTime > 0 ? ((endTime > 0 ? endTime : now) - startTime) : -1;
    final MutableIOMetrics ioMetrics = new MutableIOMetrics();
    ioMetrics.addIOMetrics(execution, metricFetcher, jobID.toString(), jobVertexID.toString());
    final IOMetricsInfo ioMetricsInfo = new IOMetricsInfo(ioMetrics.getNumBytesIn(), ioMetrics.isNumBytesInComplete(), ioMetrics.getNumBytesOut(), ioMetrics.isNumBytesOutComplete(), ioMetrics.getNumRecordsIn(), ioMetrics.isNumRecordsInComplete(), ioMetrics.getNumRecordsOut(), ioMetrics.isNumRecordsOutComplete());
    return new SubtaskExecutionAttemptDetailsInfo(execution.getParallelSubtaskIndex(), status, execution.getAttemptNumber(), locationString, startTime, endTime, duration, ioMetricsInfo, taskmanagerId);
}