Examples with SubtasksAllAccumulatorsHandler org.apache.flink.runtime.rest.handler.job.SubtasksAllAccumulatorsHandler used on opensource projects

Example 1 with SubtasksAllAccumulatorsHandler

use of org.apache.flink.runtime.rest.handler.job.SubtasksAllAccumulatorsHandler in project flink by apache.

the class WebMonitorEndpoint method initializeHandlers.

@Override
protected List<Tuple2<RestHandlerSpecification, ChannelInboundHandler>> initializeHandlers(final CompletableFuture<String> localAddressFuture) {
    ArrayList<Tuple2<RestHandlerSpecification, ChannelInboundHandler>> handlers = new ArrayList<>(30);
    final Collection<Tuple2<RestHandlerSpecification, ChannelInboundHandler>> webSubmissionHandlers = initializeWebSubmissionHandlers(localAddressFuture);
    handlers.addAll(webSubmissionHandlers);
    final boolean hasWebSubmissionHandlers = !webSubmissionHandlers.isEmpty();
    final Duration asyncOperationStoreDuration = clusterConfiguration.get(RestOptions.ASYNC_OPERATION_STORE_DURATION);
    final Time timeout = restConfiguration.getTimeout();
    ClusterOverviewHandler clusterOverviewHandler = new ClusterOverviewHandler(leaderRetriever, timeout, responseHeaders, ClusterOverviewHeaders.getInstance());
    DashboardConfigHandler dashboardConfigHandler = new DashboardConfigHandler(leaderRetriever, timeout, responseHeaders, DashboardConfigurationHeaders.getInstance(), restConfiguration.getRefreshInterval(), hasWebSubmissionHandlers, restConfiguration.isWebCancelEnabled());
    JobIdsHandler jobIdsHandler = new JobIdsHandler(leaderRetriever, timeout, responseHeaders, JobIdsWithStatusesOverviewHeaders.getInstance());
    JobsOverviewHandler jobsOverviewHandler = new JobsOverviewHandler(leaderRetriever, timeout, responseHeaders, JobsOverviewHeaders.getInstance());
    ClusterConfigHandler clusterConfigurationHandler = new ClusterConfigHandler(leaderRetriever, timeout, responseHeaders, ClusterConfigurationInfoHeaders.getInstance(), clusterConfiguration);
    JobConfigHandler jobConfigHandler = new JobConfigHandler(leaderRetriever, timeout, responseHeaders, JobConfigHeaders.getInstance(), executionGraphCache, executor);
    CheckpointConfigHandler checkpointConfigHandler = new CheckpointConfigHandler(leaderRetriever, timeout, responseHeaders, CheckpointConfigHeaders.getInstance(), executionGraphCache, executor);
    CheckpointingStatisticsHandler checkpointStatisticsHandler = new CheckpointingStatisticsHandler(leaderRetriever, timeout, responseHeaders, CheckpointingStatisticsHeaders.getInstance(), executionGraphCache, executor);
    CheckpointStatisticDetailsHandler checkpointStatisticDetailsHandler = new CheckpointStatisticDetailsHandler(leaderRetriever, timeout, responseHeaders, CheckpointStatisticDetailsHeaders.getInstance(), executionGraphCache, executor, checkpointStatsCache);
    JobPlanHandler jobPlanHandler = new JobPlanHandler(leaderRetriever, timeout, responseHeaders, JobPlanHeaders.getInstance(), executionGraphCache, executor);
    TaskCheckpointStatisticDetailsHandler taskCheckpointStatisticDetailsHandler = new TaskCheckpointStatisticDetailsHandler(leaderRetriever, timeout, responseHeaders, TaskCheckpointStatisticsHeaders.getInstance(), executionGraphCache, executor, checkpointStatsCache);
    JobExceptionsHandler jobExceptionsHandler = new JobExceptionsHandler(leaderRetriever, timeout, responseHeaders, JobExceptionsHeaders.getInstance(), executionGraphCache, executor);
    JobVertexAccumulatorsHandler jobVertexAccumulatorsHandler = new JobVertexAccumulatorsHandler(leaderRetriever, timeout, responseHeaders, JobVertexAccumulatorsHeaders.getInstance(), executionGraphCache, executor);
    SubtasksAllAccumulatorsHandler subtasksAllAccumulatorsHandler = new SubtasksAllAccumulatorsHandler(leaderRetriever, timeout, responseHeaders, SubtasksAllAccumulatorsHeaders.getInstance(), executionGraphCache, executor);
    TaskManagersHandler taskManagersHandler = new TaskManagersHandler(leaderRetriever, timeout, responseHeaders, TaskManagersHeaders.getInstance(), resourceManagerRetriever);
    TaskManagerDetailsHandler taskManagerDetailsHandler = new TaskManagerDetailsHandler(leaderRetriever, timeout, responseHeaders, TaskManagerDetailsHeaders.getInstance(), resourceManagerRetriever, metricFetcher);
    final JobDetailsHandler jobDetailsHandler = new JobDetailsHandler(leaderRetriever, timeout, responseHeaders, JobDetailsHeaders.getInstance(), executionGraphCache, executor, metricFetcher);
    JobAccumulatorsHandler jobAccumulatorsHandler = new JobAccumulatorsHandler(leaderRetriever, timeout, responseHeaders, JobAccumulatorsHeaders.getInstance(), executionGraphCache, executor);
    SubtasksTimesHandler subtasksTimesHandler = new SubtasksTimesHandler(leaderRetriever, timeout, responseHeaders, SubtasksTimesHeaders.getInstance(), executionGraphCache, executor);
    final JobVertexMetricsHandler jobVertexMetricsHandler = new JobVertexMetricsHandler(leaderRetriever, timeout, responseHeaders, metricFetcher);
    final JobVertexWatermarksHandler jobVertexWatermarksHandler = new JobVertexWatermarksHandler(leaderRetriever, timeout, responseHeaders, metricFetcher, executionGraphCache, executor);
    final JobMetricsHandler jobMetricsHandler = new JobMetricsHandler(leaderRetriever, timeout, responseHeaders, metricFetcher);
    final SubtaskMetricsHandler subtaskMetricsHandler = new SubtaskMetricsHandler(leaderRetriever, timeout, responseHeaders, metricFetcher);
    final TaskManagerMetricsHandler taskManagerMetricsHandler = new TaskManagerMetricsHandler(leaderRetriever, timeout, responseHeaders, metricFetcher);
    final JobManagerMetricsHandler jobManagerMetricsHandler = new JobManagerMetricsHandler(leaderRetriever, timeout, responseHeaders, metricFetcher);
    final AggregatingTaskManagersMetricsHandler aggregatingTaskManagersMetricsHandler = new AggregatingTaskManagersMetricsHandler(leaderRetriever, timeout, responseHeaders, executor, metricFetcher);
    final AggregatingJobsMetricsHandler aggregatingJobsMetricsHandler = new AggregatingJobsMetricsHandler(leaderRetriever, timeout, responseHeaders, executor, metricFetcher);
    final AggregatingSubtasksMetricsHandler aggregatingSubtasksMetricsHandler = new AggregatingSubtasksMetricsHandler(leaderRetriever, timeout, responseHeaders, executor, metricFetcher);
    final JobVertexTaskManagersHandler jobVertexTaskManagersHandler = new JobVertexTaskManagersHandler(leaderRetriever, timeout, responseHeaders, JobVertexTaskManagersHeaders.getInstance(), executionGraphCache, executor, metricFetcher);
    final JobExecutionResultHandler jobExecutionResultHandler = new JobExecutionResultHandler(leaderRetriever, timeout, responseHeaders);
    final String defaultSavepointDir = clusterConfiguration.getString(CheckpointingOptions.SAVEPOINT_DIRECTORY);
    final SavepointHandlers savepointHandlers = new SavepointHandlers(defaultSavepointDir);
    final SavepointHandlers.StopWithSavepointHandler stopWithSavepointHandler = savepointHandlers.new StopWithSavepointHandler(leaderRetriever, timeout, responseHeaders);
    final SavepointHandlers.SavepointTriggerHandler savepointTriggerHandler = savepointHandlers.new SavepointTriggerHandler(leaderRetriever, timeout, responseHeaders);
    final SavepointHandlers.SavepointStatusHandler savepointStatusHandler = new SavepointHandlers.SavepointStatusHandler(leaderRetriever, timeout, responseHeaders);
    final SubtaskExecutionAttemptDetailsHandler subtaskExecutionAttemptDetailsHandler = new SubtaskExecutionAttemptDetailsHandler(leaderRetriever, timeout, responseHeaders, SubtaskExecutionAttemptDetailsHeaders.getInstance(), executionGraphCache, executor, metricFetcher);
    final SubtaskExecutionAttemptAccumulatorsHandler subtaskExecutionAttemptAccumulatorsHandler = new SubtaskExecutionAttemptAccumulatorsHandler(leaderRetriever, timeout, responseHeaders, SubtaskExecutionAttemptAccumulatorsHeaders.getInstance(), executionGraphCache, executor);
    final SubtaskCurrentAttemptDetailsHandler subtaskCurrentAttemptDetailsHandler = new SubtaskCurrentAttemptDetailsHandler(leaderRetriever, timeout, responseHeaders, SubtaskCurrentAttemptDetailsHeaders.getInstance(), executionGraphCache, executor, metricFetcher);
    final RescalingHandlers rescalingHandlers = new RescalingHandlers(asyncOperationStoreDuration);
    final RescalingHandlers.RescalingTriggerHandler rescalingTriggerHandler = rescalingHandlers.new RescalingTriggerHandler(leaderRetriever, timeout, responseHeaders);
    final RescalingHandlers.RescalingStatusHandler rescalingStatusHandler = rescalingHandlers.new RescalingStatusHandler(leaderRetriever, timeout, responseHeaders);
    final JobVertexBackPressureHandler jobVertexBackPressureHandler = new JobVertexBackPressureHandler(leaderRetriever, timeout, responseHeaders, JobVertexBackPressureHeaders.getInstance(), metricFetcher);
    final JobCancellationHandler jobCancelTerminationHandler = new JobCancellationHandler(leaderRetriever, timeout, responseHeaders, JobCancellationHeaders.getInstance(), TerminationModeQueryParameter.TerminationMode.CANCEL);
    // use a separate handler for the yarn-cancel to ensure close() is only called once
    final JobCancellationHandler yarnJobCancelTerminationHandler = new JobCancellationHandler(leaderRetriever, timeout, responseHeaders, JobCancellationHeaders.getInstance(), TerminationModeQueryParameter.TerminationMode.CANCEL);
    // this is kept just for legacy reasons. STOP has been replaced by STOP-WITH-SAVEPOINT.
    final JobCancellationHandler jobStopTerminationHandler = new JobCancellationHandler(leaderRetriever, timeout, responseHeaders, JobCancellationHeaders.getInstance(), TerminationModeQueryParameter.TerminationMode.STOP);
    final JobVertexDetailsHandler jobVertexDetailsHandler = new JobVertexDetailsHandler(leaderRetriever, timeout, responseHeaders, JobVertexDetailsHeaders.getInstance(), executionGraphCache, executor, metricFetcher);
    final SavepointDisposalHandlers savepointDisposalHandlers = new SavepointDisposalHandlers(asyncOperationStoreDuration);
    final SavepointDisposalHandlers.SavepointDisposalTriggerHandler savepointDisposalTriggerHandler = savepointDisposalHandlers.new SavepointDisposalTriggerHandler(leaderRetriever, timeout, responseHeaders);
    final SavepointDisposalHandlers.SavepointDisposalStatusHandler savepointDisposalStatusHandler = savepointDisposalHandlers.new SavepointDisposalStatusHandler(leaderRetriever, timeout, responseHeaders);
    final ClusterDataSetListHandler clusterDataSetListHandler = new ClusterDataSetListHandler(leaderRetriever, timeout, responseHeaders, resourceManagerRetriever);
    final ClusterDataSetDeleteHandlers clusterDataSetDeleteHandlers = new ClusterDataSetDeleteHandlers(asyncOperationStoreDuration);
    final ClusterDataSetDeleteHandlers.ClusterDataSetDeleteTriggerHandler clusterDataSetDeleteTriggerHandler = clusterDataSetDeleteHandlers.new ClusterDataSetDeleteTriggerHandler(leaderRetriever, timeout, responseHeaders, resourceManagerRetriever);
    final ClusterDataSetDeleteHandlers.ClusterDataSetDeleteStatusHandler clusterDataSetDeleteStatusHandler = clusterDataSetDeleteHandlers.new ClusterDataSetDeleteStatusHandler(leaderRetriever, timeout, responseHeaders);
    final ClientCoordinationHandler clientCoordinationHandler = new ClientCoordinationHandler(leaderRetriever, timeout, responseHeaders, ClientCoordinationHeaders.getInstance());
    final ShutdownHandler shutdownHandler = new ShutdownHandler(leaderRetriever, timeout, responseHeaders, ShutdownHeaders.getInstance());
    final File webUiDir = restConfiguration.getWebUiDir();
    Optional<StaticFileServerHandler<T>> optWebContent;
    try {
        optWebContent = WebMonitorUtils.tryLoadWebContent(leaderRetriever, timeout, webUiDir);
    } catch (IOException e) {
        log.warn("Could not load web content handler.", e);
        optWebContent = Optional.empty();
    }
    handlers.add(Tuple2.of(clusterOverviewHandler.getMessageHeaders(), clusterOverviewHandler));
    handlers.add(Tuple2.of(clusterConfigurationHandler.getMessageHeaders(), clusterConfigurationHandler));
    handlers.add(Tuple2.of(dashboardConfigHandler.getMessageHeaders(), dashboardConfigHandler));
    handlers.add(Tuple2.of(jobIdsHandler.getMessageHeaders(), jobIdsHandler));
    handlers.add(Tuple2.of(jobsOverviewHandler.getMessageHeaders(), jobsOverviewHandler));
    handlers.add(Tuple2.of(jobConfigHandler.getMessageHeaders(), jobConfigHandler));
    handlers.add(Tuple2.of(checkpointConfigHandler.getMessageHeaders(), checkpointConfigHandler));
    handlers.add(Tuple2.of(checkpointStatisticsHandler.getMessageHeaders(), checkpointStatisticsHandler));
    handlers.add(Tuple2.of(checkpointStatisticDetailsHandler.getMessageHeaders(), checkpointStatisticDetailsHandler));
    handlers.add(Tuple2.of(jobPlanHandler.getMessageHeaders(), jobPlanHandler));
    handlers.add(Tuple2.of(taskCheckpointStatisticDetailsHandler.getMessageHeaders(), taskCheckpointStatisticDetailsHandler));
    handlers.add(Tuple2.of(jobExceptionsHandler.getMessageHeaders(), jobExceptionsHandler));
    handlers.add(Tuple2.of(jobVertexAccumulatorsHandler.getMessageHeaders(), jobVertexAccumulatorsHandler));
    handlers.add(Tuple2.of(subtasksAllAccumulatorsHandler.getMessageHeaders(), subtasksAllAccumulatorsHandler));
    handlers.add(Tuple2.of(jobDetailsHandler.getMessageHeaders(), jobDetailsHandler));
    handlers.add(Tuple2.of(jobAccumulatorsHandler.getMessageHeaders(), jobAccumulatorsHandler));
    handlers.add(Tuple2.of(taskManagersHandler.getMessageHeaders(), taskManagersHandler));
    handlers.add(Tuple2.of(taskManagerDetailsHandler.getMessageHeaders(), taskManagerDetailsHandler));
    handlers.add(Tuple2.of(subtasksTimesHandler.getMessageHeaders(), subtasksTimesHandler));
    handlers.add(Tuple2.of(jobVertexMetricsHandler.getMessageHeaders(), jobVertexMetricsHandler));
    handlers.add(Tuple2.of(jobVertexWatermarksHandler.getMessageHeaders(), jobVertexWatermarksHandler));
    handlers.add(Tuple2.of(jobMetricsHandler.getMessageHeaders(), jobMetricsHandler));
    handlers.add(Tuple2.of(subtaskMetricsHandler.getMessageHeaders(), subtaskMetricsHandler));
    handlers.add(Tuple2.of(taskManagerMetricsHandler.getMessageHeaders(), taskManagerMetricsHandler));
    handlers.add(Tuple2.of(jobManagerMetricsHandler.getMessageHeaders(), jobManagerMetricsHandler));
    handlers.add(Tuple2.of(aggregatingTaskManagersMetricsHandler.getMessageHeaders(), aggregatingTaskManagersMetricsHandler));
    handlers.add(Tuple2.of(aggregatingJobsMetricsHandler.getMessageHeaders(), aggregatingJobsMetricsHandler));
    handlers.add(Tuple2.of(aggregatingSubtasksMetricsHandler.getMessageHeaders(), aggregatingSubtasksMetricsHandler));
    handlers.add(Tuple2.of(jobExecutionResultHandler.getMessageHeaders(), jobExecutionResultHandler));
    handlers.add(Tuple2.of(savepointTriggerHandler.getMessageHeaders(), savepointTriggerHandler));
    handlers.add(Tuple2.of(stopWithSavepointHandler.getMessageHeaders(), stopWithSavepointHandler));
    handlers.add(Tuple2.of(savepointStatusHandler.getMessageHeaders(), savepointStatusHandler));
    handlers.add(Tuple2.of(subtaskExecutionAttemptDetailsHandler.getMessageHeaders(), subtaskExecutionAttemptDetailsHandler));
    handlers.add(Tuple2.of(subtaskExecutionAttemptAccumulatorsHandler.getMessageHeaders(), subtaskExecutionAttemptAccumulatorsHandler));
    handlers.add(Tuple2.of(subtaskCurrentAttemptDetailsHandler.getMessageHeaders(), subtaskCurrentAttemptDetailsHandler));
    handlers.add(Tuple2.of(jobVertexTaskManagersHandler.getMessageHeaders(), jobVertexTaskManagersHandler));
    handlers.add(Tuple2.of(jobVertexBackPressureHandler.getMessageHeaders(), jobVertexBackPressureHandler));
    final AbstractRestHandler<?, ?, ?, ?> jobVertexFlameGraphHandler;
    if (clusterConfiguration.get(RestOptions.ENABLE_FLAMEGRAPH)) {
        jobVertexFlameGraphHandler = new JobVertexFlameGraphHandler(leaderRetriever, timeout, responseHeaders, executionGraphCache, executor, initializeThreadInfoTracker(executor));
    } else {
        jobVertexFlameGraphHandler = JobVertexFlameGraphHandler.disabledHandler(leaderRetriever, timeout, responseHeaders);
    }
    handlers.add(Tuple2.of(jobVertexFlameGraphHandler.getMessageHeaders(), jobVertexFlameGraphHandler));
    handlers.add(Tuple2.of(jobCancelTerminationHandler.getMessageHeaders(), jobCancelTerminationHandler));
    handlers.add(Tuple2.of(jobVertexDetailsHandler.getMessageHeaders(), jobVertexDetailsHandler));
    handlers.add(Tuple2.of(rescalingTriggerHandler.getMessageHeaders(), rescalingTriggerHandler));
    handlers.add(Tuple2.of(rescalingStatusHandler.getMessageHeaders(), rescalingStatusHandler));
    handlers.add(Tuple2.of(savepointDisposalTriggerHandler.getMessageHeaders(), savepointDisposalTriggerHandler));
    handlers.add(Tuple2.of(savepointDisposalStatusHandler.getMessageHeaders(), savepointDisposalStatusHandler));
    handlers.add(Tuple2.of(clusterDataSetListHandler.getMessageHeaders(), clusterDataSetListHandler));
    handlers.add(Tuple2.of(clusterDataSetDeleteTriggerHandler.getMessageHeaders(), clusterDataSetDeleteTriggerHandler));
    handlers.add(Tuple2.of(clusterDataSetDeleteStatusHandler.getMessageHeaders(), clusterDataSetDeleteStatusHandler));
    handlers.add(Tuple2.of(clientCoordinationHandler.getMessageHeaders(), clientCoordinationHandler));
    // TODO: Remove once the Yarn proxy can forward all REST verbs
    handlers.add(Tuple2.of(YarnCancelJobTerminationHeaders.getInstance(), yarnJobCancelTerminationHandler));
    handlers.add(Tuple2.of(YarnStopJobTerminationHeaders.getInstance(), jobStopTerminationHandler));
    handlers.add(Tuple2.of(shutdownHandler.getMessageHeaders(), shutdownHandler));
    optWebContent.ifPresent(webContent -> {
        handlers.add(Tuple2.of(WebContentHandlerSpecification.getInstance(), webContent));
        hasWebUI = true;
    });
    // load the log and stdout file handler for the main cluster component
    final WebMonitorUtils.LogFileLocation logFileLocation = WebMonitorUtils.LogFileLocation.find(clusterConfiguration);
    final JobManagerLogFileHandler jobManagerLogFileHandler = new JobManagerLogFileHandler(leaderRetriever, timeout, responseHeaders, JobManagerLogFileHeader.getInstance(), logFileLocation.logFile);
    final JobManagerLogFileHandler jobManagerStdoutFileHandler = new JobManagerLogFileHandler(leaderRetriever, timeout, responseHeaders, JobManagerStdoutFileHeader.getInstance(), logFileLocation.stdOutFile);
    final JobManagerCustomLogHandler jobManagerCustomLogHandler = new JobManagerCustomLogHandler(leaderRetriever, timeout, responseHeaders, JobManagerCustomLogHeaders.getInstance(), logFileLocation.logDir);
    final JobManagerLogListHandler jobManagerLogListHandler = new JobManagerLogListHandler(leaderRetriever, timeout, responseHeaders, JobManagerLogListHeaders.getInstance(), logFileLocation.logDir);
    final JobManagerThreadDumpHandler jobManagerThreadDumpHandler = new JobManagerThreadDumpHandler(leaderRetriever, timeout, responseHeaders, JobManagerThreadDumpHeaders.getInstance());
    handlers.add(Tuple2.of(JobManagerLogFileHeader.getInstance(), jobManagerLogFileHandler));
    handlers.add(Tuple2.of(JobManagerStdoutFileHeader.getInstance(), jobManagerStdoutFileHandler));
    handlers.add(Tuple2.of(JobManagerCustomLogHeaders.getInstance(), jobManagerCustomLogHandler));
    handlers.add(Tuple2.of(JobManagerLogListHeaders.getInstance(), jobManagerLogListHandler));
    handlers.add(Tuple2.of(JobManagerThreadDumpHeaders.getInstance(), jobManagerThreadDumpHandler));
    // TaskManager log and stdout file handler
    final Time cacheEntryDuration = Time.milliseconds(restConfiguration.getRefreshInterval());
    final TaskManagerLogFileHandler taskManagerLogFileHandler = new TaskManagerLogFileHandler(leaderRetriever, timeout, responseHeaders, TaskManagerLogFileHeaders.getInstance(), resourceManagerRetriever, transientBlobService, cacheEntryDuration);
    final TaskManagerStdoutFileHandler taskManagerStdoutFileHandler = new TaskManagerStdoutFileHandler(leaderRetriever, timeout, responseHeaders, TaskManagerStdoutFileHeaders.getInstance(), resourceManagerRetriever, transientBlobService, cacheEntryDuration);
    final TaskManagerCustomLogHandler taskManagerCustomLogHandler = new TaskManagerCustomLogHandler(leaderRetriever, timeout, responseHeaders, TaskManagerCustomLogHeaders.getInstance(), resourceManagerRetriever, transientBlobService, cacheEntryDuration);
    final TaskManagerLogListHandler taskManagerLogListHandler = new TaskManagerLogListHandler(leaderRetriever, timeout, responseHeaders, TaskManagerLogsHeaders.getInstance(), resourceManagerRetriever);
    final TaskManagerThreadDumpHandler taskManagerThreadDumpFileHandler = new TaskManagerThreadDumpHandler(leaderRetriever, timeout, responseHeaders, TaskManagerThreadDumpHeaders.getInstance(), resourceManagerRetriever);
    handlers.add(Tuple2.of(TaskManagerLogFileHeaders.getInstance(), taskManagerLogFileHandler));
    handlers.add(Tuple2.of(TaskManagerStdoutFileHeaders.getInstance(), taskManagerStdoutFileHandler));
    handlers.add(Tuple2.of(TaskManagerCustomLogHeaders.getInstance(), taskManagerCustomLogHandler));
    handlers.add(Tuple2.of(TaskManagerLogsHeaders.getInstance(), taskManagerLogListHandler));
    handlers.add(Tuple2.of(TaskManagerThreadDumpHeaders.getInstance(), taskManagerThreadDumpFileHandler));
    handlers.stream().map(tuple -> tuple.f1).filter(handler -> handler instanceof JsonArchivist).forEachOrdered(handler -> archivingHandlers.add((JsonArchivist) handler));
    return handlers;
}