Search in sources :

Example 1 with TransientBlobService

use of org.apache.flink.runtime.blob.TransientBlobService in project flink by apache.

the class TaskExecutor method putTransientBlobStream.

private CompletableFuture<TransientBlobKey> putTransientBlobStream(InputStream inputStream, String fileTag) {
    final TransientBlobService transientBlobService = taskExecutorBlobService.getTransientBlobService();
    final TransientBlobKey transientBlobKey;
    try {
        transientBlobKey = transientBlobService.putTransient(inputStream);
    } catch (IOException e) {
        log.debug("Could not upload file {}.", fileTag, e);
        return FutureUtils.completedExceptionally(new FlinkException("Could not upload file " + fileTag + '.', e));
    }
    return CompletableFuture.completedFuture(transientBlobKey);
}
Also used : TransientBlobService(org.apache.flink.runtime.blob.TransientBlobService) TransientBlobKey(org.apache.flink.runtime.blob.TransientBlobKey) IOException(java.io.IOException) FlinkException(org.apache.flink.util.FlinkException)

Example 2 with TransientBlobService

use of org.apache.flink.runtime.blob.TransientBlobService in project flink by apache.

the class WebMonitorEndpoint method initializeHandlers.

@Override
protected List<Tuple2<RestHandlerSpecification, ChannelInboundHandler>> initializeHandlers(final CompletableFuture<String> localAddressFuture) {
    ArrayList<Tuple2<RestHandlerSpecification, ChannelInboundHandler>> handlers = new ArrayList<>(30);
    final Collection<Tuple2<RestHandlerSpecification, ChannelInboundHandler>> webSubmissionHandlers = initializeWebSubmissionHandlers(localAddressFuture);
    handlers.addAll(webSubmissionHandlers);
    final boolean hasWebSubmissionHandlers = !webSubmissionHandlers.isEmpty();
    final Duration asyncOperationStoreDuration = clusterConfiguration.get(RestOptions.ASYNC_OPERATION_STORE_DURATION);
    final Time timeout = restConfiguration.getTimeout();
    ClusterOverviewHandler clusterOverviewHandler = new ClusterOverviewHandler(leaderRetriever, timeout, responseHeaders, ClusterOverviewHeaders.getInstance());
    DashboardConfigHandler dashboardConfigHandler = new DashboardConfigHandler(leaderRetriever, timeout, responseHeaders, DashboardConfigurationHeaders.getInstance(), restConfiguration.getRefreshInterval(), hasWebSubmissionHandlers, restConfiguration.isWebCancelEnabled());
    JobIdsHandler jobIdsHandler = new JobIdsHandler(leaderRetriever, timeout, responseHeaders, JobIdsWithStatusesOverviewHeaders.getInstance());
    JobsOverviewHandler jobsOverviewHandler = new JobsOverviewHandler(leaderRetriever, timeout, responseHeaders, JobsOverviewHeaders.getInstance());
    ClusterConfigHandler clusterConfigurationHandler = new ClusterConfigHandler(leaderRetriever, timeout, responseHeaders, ClusterConfigurationInfoHeaders.getInstance(), clusterConfiguration);
    JobConfigHandler jobConfigHandler = new JobConfigHandler(leaderRetriever, timeout, responseHeaders, JobConfigHeaders.getInstance(), executionGraphCache, executor);
    CheckpointConfigHandler checkpointConfigHandler = new CheckpointConfigHandler(leaderRetriever, timeout, responseHeaders, CheckpointConfigHeaders.getInstance(), executionGraphCache, executor);
    CheckpointingStatisticsHandler checkpointStatisticsHandler = new CheckpointingStatisticsHandler(leaderRetriever, timeout, responseHeaders, CheckpointingStatisticsHeaders.getInstance(), executionGraphCache, executor);
    CheckpointStatisticDetailsHandler checkpointStatisticDetailsHandler = new CheckpointStatisticDetailsHandler(leaderRetriever, timeout, responseHeaders, CheckpointStatisticDetailsHeaders.getInstance(), executionGraphCache, executor, checkpointStatsCache);
    JobPlanHandler jobPlanHandler = new JobPlanHandler(leaderRetriever, timeout, responseHeaders, JobPlanHeaders.getInstance(), executionGraphCache, executor);
    TaskCheckpointStatisticDetailsHandler taskCheckpointStatisticDetailsHandler = new TaskCheckpointStatisticDetailsHandler(leaderRetriever, timeout, responseHeaders, TaskCheckpointStatisticsHeaders.getInstance(), executionGraphCache, executor, checkpointStatsCache);
    JobExceptionsHandler jobExceptionsHandler = new JobExceptionsHandler(leaderRetriever, timeout, responseHeaders, JobExceptionsHeaders.getInstance(), executionGraphCache, executor);
    JobVertexAccumulatorsHandler jobVertexAccumulatorsHandler = new JobVertexAccumulatorsHandler(leaderRetriever, timeout, responseHeaders, JobVertexAccumulatorsHeaders.getInstance(), executionGraphCache, executor);
    SubtasksAllAccumulatorsHandler subtasksAllAccumulatorsHandler = new SubtasksAllAccumulatorsHandler(leaderRetriever, timeout, responseHeaders, SubtasksAllAccumulatorsHeaders.getInstance(), executionGraphCache, executor);
    TaskManagersHandler taskManagersHandler = new TaskManagersHandler(leaderRetriever, timeout, responseHeaders, TaskManagersHeaders.getInstance(), resourceManagerRetriever);
    TaskManagerDetailsHandler taskManagerDetailsHandler = new TaskManagerDetailsHandler(leaderRetriever, timeout, responseHeaders, TaskManagerDetailsHeaders.getInstance(), resourceManagerRetriever, metricFetcher);
    final JobDetailsHandler jobDetailsHandler = new JobDetailsHandler(leaderRetriever, timeout, responseHeaders, JobDetailsHeaders.getInstance(), executionGraphCache, executor, metricFetcher);
    JobAccumulatorsHandler jobAccumulatorsHandler = new JobAccumulatorsHandler(leaderRetriever, timeout, responseHeaders, JobAccumulatorsHeaders.getInstance(), executionGraphCache, executor);
    SubtasksTimesHandler subtasksTimesHandler = new SubtasksTimesHandler(leaderRetriever, timeout, responseHeaders, SubtasksTimesHeaders.getInstance(), executionGraphCache, executor);
    final JobVertexMetricsHandler jobVertexMetricsHandler = new JobVertexMetricsHandler(leaderRetriever, timeout, responseHeaders, metricFetcher);
    final JobVertexWatermarksHandler jobVertexWatermarksHandler = new JobVertexWatermarksHandler(leaderRetriever, timeout, responseHeaders, metricFetcher, executionGraphCache, executor);
    final JobMetricsHandler jobMetricsHandler = new JobMetricsHandler(leaderRetriever, timeout, responseHeaders, metricFetcher);
    final SubtaskMetricsHandler subtaskMetricsHandler = new SubtaskMetricsHandler(leaderRetriever, timeout, responseHeaders, metricFetcher);
    final TaskManagerMetricsHandler taskManagerMetricsHandler = new TaskManagerMetricsHandler(leaderRetriever, timeout, responseHeaders, metricFetcher);
    final JobManagerMetricsHandler jobManagerMetricsHandler = new JobManagerMetricsHandler(leaderRetriever, timeout, responseHeaders, metricFetcher);
    final AggregatingTaskManagersMetricsHandler aggregatingTaskManagersMetricsHandler = new AggregatingTaskManagersMetricsHandler(leaderRetriever, timeout, responseHeaders, executor, metricFetcher);
    final AggregatingJobsMetricsHandler aggregatingJobsMetricsHandler = new AggregatingJobsMetricsHandler(leaderRetriever, timeout, responseHeaders, executor, metricFetcher);
    final AggregatingSubtasksMetricsHandler aggregatingSubtasksMetricsHandler = new AggregatingSubtasksMetricsHandler(leaderRetriever, timeout, responseHeaders, executor, metricFetcher);
    final JobVertexTaskManagersHandler jobVertexTaskManagersHandler = new JobVertexTaskManagersHandler(leaderRetriever, timeout, responseHeaders, JobVertexTaskManagersHeaders.getInstance(), executionGraphCache, executor, metricFetcher);
    final JobExecutionResultHandler jobExecutionResultHandler = new JobExecutionResultHandler(leaderRetriever, timeout, responseHeaders);
    final String defaultSavepointDir = clusterConfiguration.getString(CheckpointingOptions.SAVEPOINT_DIRECTORY);
    final SavepointHandlers savepointHandlers = new SavepointHandlers(defaultSavepointDir);
    final SavepointHandlers.StopWithSavepointHandler stopWithSavepointHandler = savepointHandlers.new StopWithSavepointHandler(leaderRetriever, timeout, responseHeaders);
    final SavepointHandlers.SavepointTriggerHandler savepointTriggerHandler = savepointHandlers.new SavepointTriggerHandler(leaderRetriever, timeout, responseHeaders);
    final SavepointHandlers.SavepointStatusHandler savepointStatusHandler = new SavepointHandlers.SavepointStatusHandler(leaderRetriever, timeout, responseHeaders);
    final SubtaskExecutionAttemptDetailsHandler subtaskExecutionAttemptDetailsHandler = new SubtaskExecutionAttemptDetailsHandler(leaderRetriever, timeout, responseHeaders, SubtaskExecutionAttemptDetailsHeaders.getInstance(), executionGraphCache, executor, metricFetcher);
    final SubtaskExecutionAttemptAccumulatorsHandler subtaskExecutionAttemptAccumulatorsHandler = new SubtaskExecutionAttemptAccumulatorsHandler(leaderRetriever, timeout, responseHeaders, SubtaskExecutionAttemptAccumulatorsHeaders.getInstance(), executionGraphCache, executor);
    final SubtaskCurrentAttemptDetailsHandler subtaskCurrentAttemptDetailsHandler = new SubtaskCurrentAttemptDetailsHandler(leaderRetriever, timeout, responseHeaders, SubtaskCurrentAttemptDetailsHeaders.getInstance(), executionGraphCache, executor, metricFetcher);
    final RescalingHandlers rescalingHandlers = new RescalingHandlers(asyncOperationStoreDuration);
    final RescalingHandlers.RescalingTriggerHandler rescalingTriggerHandler = rescalingHandlers.new RescalingTriggerHandler(leaderRetriever, timeout, responseHeaders);
    final RescalingHandlers.RescalingStatusHandler rescalingStatusHandler = rescalingHandlers.new RescalingStatusHandler(leaderRetriever, timeout, responseHeaders);
    final JobVertexBackPressureHandler jobVertexBackPressureHandler = new JobVertexBackPressureHandler(leaderRetriever, timeout, responseHeaders, JobVertexBackPressureHeaders.getInstance(), metricFetcher);
    final JobCancellationHandler jobCancelTerminationHandler = new JobCancellationHandler(leaderRetriever, timeout, responseHeaders, JobCancellationHeaders.getInstance(), TerminationModeQueryParameter.TerminationMode.CANCEL);
    // use a separate handler for the yarn-cancel to ensure close() is only called once
    final JobCancellationHandler yarnJobCancelTerminationHandler = new JobCancellationHandler(leaderRetriever, timeout, responseHeaders, JobCancellationHeaders.getInstance(), TerminationModeQueryParameter.TerminationMode.CANCEL);
    // this is kept just for legacy reasons. STOP has been replaced by STOP-WITH-SAVEPOINT.
    final JobCancellationHandler jobStopTerminationHandler = new JobCancellationHandler(leaderRetriever, timeout, responseHeaders, JobCancellationHeaders.getInstance(), TerminationModeQueryParameter.TerminationMode.STOP);
    final JobVertexDetailsHandler jobVertexDetailsHandler = new JobVertexDetailsHandler(leaderRetriever, timeout, responseHeaders, JobVertexDetailsHeaders.getInstance(), executionGraphCache, executor, metricFetcher);
    final SavepointDisposalHandlers savepointDisposalHandlers = new SavepointDisposalHandlers(asyncOperationStoreDuration);
    final SavepointDisposalHandlers.SavepointDisposalTriggerHandler savepointDisposalTriggerHandler = savepointDisposalHandlers.new SavepointDisposalTriggerHandler(leaderRetriever, timeout, responseHeaders);
    final SavepointDisposalHandlers.SavepointDisposalStatusHandler savepointDisposalStatusHandler = savepointDisposalHandlers.new SavepointDisposalStatusHandler(leaderRetriever, timeout, responseHeaders);
    final ClusterDataSetListHandler clusterDataSetListHandler = new ClusterDataSetListHandler(leaderRetriever, timeout, responseHeaders, resourceManagerRetriever);
    final ClusterDataSetDeleteHandlers clusterDataSetDeleteHandlers = new ClusterDataSetDeleteHandlers(asyncOperationStoreDuration);
    final ClusterDataSetDeleteHandlers.ClusterDataSetDeleteTriggerHandler clusterDataSetDeleteTriggerHandler = clusterDataSetDeleteHandlers.new ClusterDataSetDeleteTriggerHandler(leaderRetriever, timeout, responseHeaders, resourceManagerRetriever);
    final ClusterDataSetDeleteHandlers.ClusterDataSetDeleteStatusHandler clusterDataSetDeleteStatusHandler = clusterDataSetDeleteHandlers.new ClusterDataSetDeleteStatusHandler(leaderRetriever, timeout, responseHeaders);
    final ClientCoordinationHandler clientCoordinationHandler = new ClientCoordinationHandler(leaderRetriever, timeout, responseHeaders, ClientCoordinationHeaders.getInstance());
    final ShutdownHandler shutdownHandler = new ShutdownHandler(leaderRetriever, timeout, responseHeaders, ShutdownHeaders.getInstance());
    final File webUiDir = restConfiguration.getWebUiDir();
    Optional<StaticFileServerHandler<T>> optWebContent;
    try {
        optWebContent = WebMonitorUtils.tryLoadWebContent(leaderRetriever, timeout, webUiDir);
    } catch (IOException e) {
        log.warn("Could not load web content handler.", e);
        optWebContent = Optional.empty();
    }
    handlers.add(Tuple2.of(clusterOverviewHandler.getMessageHeaders(), clusterOverviewHandler));
    handlers.add(Tuple2.of(clusterConfigurationHandler.getMessageHeaders(), clusterConfigurationHandler));
    handlers.add(Tuple2.of(dashboardConfigHandler.getMessageHeaders(), dashboardConfigHandler));
    handlers.add(Tuple2.of(jobIdsHandler.getMessageHeaders(), jobIdsHandler));
    handlers.add(Tuple2.of(jobsOverviewHandler.getMessageHeaders(), jobsOverviewHandler));
    handlers.add(Tuple2.of(jobConfigHandler.getMessageHeaders(), jobConfigHandler));
    handlers.add(Tuple2.of(checkpointConfigHandler.getMessageHeaders(), checkpointConfigHandler));
    handlers.add(Tuple2.of(checkpointStatisticsHandler.getMessageHeaders(), checkpointStatisticsHandler));
    handlers.add(Tuple2.of(checkpointStatisticDetailsHandler.getMessageHeaders(), checkpointStatisticDetailsHandler));
    handlers.add(Tuple2.of(jobPlanHandler.getMessageHeaders(), jobPlanHandler));
    handlers.add(Tuple2.of(taskCheckpointStatisticDetailsHandler.getMessageHeaders(), taskCheckpointStatisticDetailsHandler));
    handlers.add(Tuple2.of(jobExceptionsHandler.getMessageHeaders(), jobExceptionsHandler));
    handlers.add(Tuple2.of(jobVertexAccumulatorsHandler.getMessageHeaders(), jobVertexAccumulatorsHandler));
    handlers.add(Tuple2.of(subtasksAllAccumulatorsHandler.getMessageHeaders(), subtasksAllAccumulatorsHandler));
    handlers.add(Tuple2.of(jobDetailsHandler.getMessageHeaders(), jobDetailsHandler));
    handlers.add(Tuple2.of(jobAccumulatorsHandler.getMessageHeaders(), jobAccumulatorsHandler));
    handlers.add(Tuple2.of(taskManagersHandler.getMessageHeaders(), taskManagersHandler));
    handlers.add(Tuple2.of(taskManagerDetailsHandler.getMessageHeaders(), taskManagerDetailsHandler));
    handlers.add(Tuple2.of(subtasksTimesHandler.getMessageHeaders(), subtasksTimesHandler));
    handlers.add(Tuple2.of(jobVertexMetricsHandler.getMessageHeaders(), jobVertexMetricsHandler));
    handlers.add(Tuple2.of(jobVertexWatermarksHandler.getMessageHeaders(), jobVertexWatermarksHandler));
    handlers.add(Tuple2.of(jobMetricsHandler.getMessageHeaders(), jobMetricsHandler));
    handlers.add(Tuple2.of(subtaskMetricsHandler.getMessageHeaders(), subtaskMetricsHandler));
    handlers.add(Tuple2.of(taskManagerMetricsHandler.getMessageHeaders(), taskManagerMetricsHandler));
    handlers.add(Tuple2.of(jobManagerMetricsHandler.getMessageHeaders(), jobManagerMetricsHandler));
    handlers.add(Tuple2.of(aggregatingTaskManagersMetricsHandler.getMessageHeaders(), aggregatingTaskManagersMetricsHandler));
    handlers.add(Tuple2.of(aggregatingJobsMetricsHandler.getMessageHeaders(), aggregatingJobsMetricsHandler));
    handlers.add(Tuple2.of(aggregatingSubtasksMetricsHandler.getMessageHeaders(), aggregatingSubtasksMetricsHandler));
    handlers.add(Tuple2.of(jobExecutionResultHandler.getMessageHeaders(), jobExecutionResultHandler));
    handlers.add(Tuple2.of(savepointTriggerHandler.getMessageHeaders(), savepointTriggerHandler));
    handlers.add(Tuple2.of(stopWithSavepointHandler.getMessageHeaders(), stopWithSavepointHandler));
    handlers.add(Tuple2.of(savepointStatusHandler.getMessageHeaders(), savepointStatusHandler));
    handlers.add(Tuple2.of(subtaskExecutionAttemptDetailsHandler.getMessageHeaders(), subtaskExecutionAttemptDetailsHandler));
    handlers.add(Tuple2.of(subtaskExecutionAttemptAccumulatorsHandler.getMessageHeaders(), subtaskExecutionAttemptAccumulatorsHandler));
    handlers.add(Tuple2.of(subtaskCurrentAttemptDetailsHandler.getMessageHeaders(), subtaskCurrentAttemptDetailsHandler));
    handlers.add(Tuple2.of(jobVertexTaskManagersHandler.getMessageHeaders(), jobVertexTaskManagersHandler));
    handlers.add(Tuple2.of(jobVertexBackPressureHandler.getMessageHeaders(), jobVertexBackPressureHandler));
    final AbstractRestHandler<?, ?, ?, ?> jobVertexFlameGraphHandler;
    if (clusterConfiguration.get(RestOptions.ENABLE_FLAMEGRAPH)) {
        jobVertexFlameGraphHandler = new JobVertexFlameGraphHandler(leaderRetriever, timeout, responseHeaders, executionGraphCache, executor, initializeThreadInfoTracker(executor));
    } else {
        jobVertexFlameGraphHandler = JobVertexFlameGraphHandler.disabledHandler(leaderRetriever, timeout, responseHeaders);
    }
    handlers.add(Tuple2.of(jobVertexFlameGraphHandler.getMessageHeaders(), jobVertexFlameGraphHandler));
    handlers.add(Tuple2.of(jobCancelTerminationHandler.getMessageHeaders(), jobCancelTerminationHandler));
    handlers.add(Tuple2.of(jobVertexDetailsHandler.getMessageHeaders(), jobVertexDetailsHandler));
    handlers.add(Tuple2.of(rescalingTriggerHandler.getMessageHeaders(), rescalingTriggerHandler));
    handlers.add(Tuple2.of(rescalingStatusHandler.getMessageHeaders(), rescalingStatusHandler));
    handlers.add(Tuple2.of(savepointDisposalTriggerHandler.getMessageHeaders(), savepointDisposalTriggerHandler));
    handlers.add(Tuple2.of(savepointDisposalStatusHandler.getMessageHeaders(), savepointDisposalStatusHandler));
    handlers.add(Tuple2.of(clusterDataSetListHandler.getMessageHeaders(), clusterDataSetListHandler));
    handlers.add(Tuple2.of(clusterDataSetDeleteTriggerHandler.getMessageHeaders(), clusterDataSetDeleteTriggerHandler));
    handlers.add(Tuple2.of(clusterDataSetDeleteStatusHandler.getMessageHeaders(), clusterDataSetDeleteStatusHandler));
    handlers.add(Tuple2.of(clientCoordinationHandler.getMessageHeaders(), clientCoordinationHandler));
    // TODO: Remove once the Yarn proxy can forward all REST verbs
    handlers.add(Tuple2.of(YarnCancelJobTerminationHeaders.getInstance(), yarnJobCancelTerminationHandler));
    handlers.add(Tuple2.of(YarnStopJobTerminationHeaders.getInstance(), jobStopTerminationHandler));
    handlers.add(Tuple2.of(shutdownHandler.getMessageHeaders(), shutdownHandler));
    optWebContent.ifPresent(webContent -> {
        handlers.add(Tuple2.of(WebContentHandlerSpecification.getInstance(), webContent));
        hasWebUI = true;
    });
    // load the log and stdout file handler for the main cluster component
    final WebMonitorUtils.LogFileLocation logFileLocation = WebMonitorUtils.LogFileLocation.find(clusterConfiguration);
    final JobManagerLogFileHandler jobManagerLogFileHandler = new JobManagerLogFileHandler(leaderRetriever, timeout, responseHeaders, JobManagerLogFileHeader.getInstance(), logFileLocation.logFile);
    final JobManagerLogFileHandler jobManagerStdoutFileHandler = new JobManagerLogFileHandler(leaderRetriever, timeout, responseHeaders, JobManagerStdoutFileHeader.getInstance(), logFileLocation.stdOutFile);
    final JobManagerCustomLogHandler jobManagerCustomLogHandler = new JobManagerCustomLogHandler(leaderRetriever, timeout, responseHeaders, JobManagerCustomLogHeaders.getInstance(), logFileLocation.logDir);
    final JobManagerLogListHandler jobManagerLogListHandler = new JobManagerLogListHandler(leaderRetriever, timeout, responseHeaders, JobManagerLogListHeaders.getInstance(), logFileLocation.logDir);
    final JobManagerThreadDumpHandler jobManagerThreadDumpHandler = new JobManagerThreadDumpHandler(leaderRetriever, timeout, responseHeaders, JobManagerThreadDumpHeaders.getInstance());
    handlers.add(Tuple2.of(JobManagerLogFileHeader.getInstance(), jobManagerLogFileHandler));
    handlers.add(Tuple2.of(JobManagerStdoutFileHeader.getInstance(), jobManagerStdoutFileHandler));
    handlers.add(Tuple2.of(JobManagerCustomLogHeaders.getInstance(), jobManagerCustomLogHandler));
    handlers.add(Tuple2.of(JobManagerLogListHeaders.getInstance(), jobManagerLogListHandler));
    handlers.add(Tuple2.of(JobManagerThreadDumpHeaders.getInstance(), jobManagerThreadDumpHandler));
    // TaskManager log and stdout file handler
    final Time cacheEntryDuration = Time.milliseconds(restConfiguration.getRefreshInterval());
    final TaskManagerLogFileHandler taskManagerLogFileHandler = new TaskManagerLogFileHandler(leaderRetriever, timeout, responseHeaders, TaskManagerLogFileHeaders.getInstance(), resourceManagerRetriever, transientBlobService, cacheEntryDuration);
    final TaskManagerStdoutFileHandler taskManagerStdoutFileHandler = new TaskManagerStdoutFileHandler(leaderRetriever, timeout, responseHeaders, TaskManagerStdoutFileHeaders.getInstance(), resourceManagerRetriever, transientBlobService, cacheEntryDuration);
    final TaskManagerCustomLogHandler taskManagerCustomLogHandler = new TaskManagerCustomLogHandler(leaderRetriever, timeout, responseHeaders, TaskManagerCustomLogHeaders.getInstance(), resourceManagerRetriever, transientBlobService, cacheEntryDuration);
    final TaskManagerLogListHandler taskManagerLogListHandler = new TaskManagerLogListHandler(leaderRetriever, timeout, responseHeaders, TaskManagerLogsHeaders.getInstance(), resourceManagerRetriever);
    final TaskManagerThreadDumpHandler taskManagerThreadDumpFileHandler = new TaskManagerThreadDumpHandler(leaderRetriever, timeout, responseHeaders, TaskManagerThreadDumpHeaders.getInstance(), resourceManagerRetriever);
    handlers.add(Tuple2.of(TaskManagerLogFileHeaders.getInstance(), taskManagerLogFileHandler));
    handlers.add(Tuple2.of(TaskManagerStdoutFileHeaders.getInstance(), taskManagerStdoutFileHandler));
    handlers.add(Tuple2.of(TaskManagerCustomLogHeaders.getInstance(), taskManagerCustomLogHandler));
    handlers.add(Tuple2.of(TaskManagerLogsHeaders.getInstance(), taskManagerLogListHandler));
    handlers.add(Tuple2.of(TaskManagerThreadDumpHeaders.getInstance(), taskManagerThreadDumpFileHandler));
    handlers.stream().map(tuple -> tuple.f1).filter(handler -> handler instanceof JsonArchivist).forEachOrdered(handler -> archivingHandlers.add((JsonArchivist) handler));
    return handlers;
}
Also used : JobManagerThreadDumpHandler(org.apache.flink.runtime.rest.handler.cluster.JobManagerThreadDumpHandler) JobAccumulatorsHeaders(org.apache.flink.runtime.rest.messages.JobAccumulatorsHeaders) TaskManagerThreadDumpHandler(org.apache.flink.runtime.rest.handler.taskmanager.TaskManagerThreadDumpHandler) CheckpointStatisticDetailsHandler(org.apache.flink.runtime.rest.handler.job.checkpoints.CheckpointStatisticDetailsHandler) Tuple2(org.apache.flink.api.java.tuple.Tuple2) ExecutionGraphCache(org.apache.flink.runtime.rest.handler.legacy.ExecutionGraphCache) TaskManagerStdoutFileHandler(org.apache.flink.runtime.rest.handler.taskmanager.TaskManagerStdoutFileHandler) ClusterOverviewHandler(org.apache.flink.runtime.rest.handler.cluster.ClusterOverviewHandler) JobExceptionsHeaders(org.apache.flink.runtime.rest.messages.JobExceptionsHeaders) JobManagerLogListHeaders(org.apache.flink.runtime.rest.messages.cluster.JobManagerLogListHeaders) TaskManagerLogFileHeaders(org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerLogFileHeaders) Duration(java.time.Duration) JobConfigHandler(org.apache.flink.runtime.rest.handler.job.JobConfigHandler) SubtaskMetricsHandler(org.apache.flink.runtime.rest.handler.job.metrics.SubtaskMetricsHandler) JobPlanHeaders(org.apache.flink.runtime.rest.messages.JobPlanHeaders) ShutdownHeaders(org.apache.flink.runtime.rest.messages.cluster.ShutdownHeaders) ClientCoordinationHeaders(org.apache.flink.runtime.rest.messages.job.coordination.ClientCoordinationHeaders) TaskManagerDetailsHeaders(org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerDetailsHeaders) ThreadInfoRequestCoordinator(org.apache.flink.runtime.webmonitor.threadinfo.ThreadInfoRequestCoordinator) TaskCheckpointStatisticsHeaders(org.apache.flink.runtime.rest.messages.checkpoints.TaskCheckpointStatisticsHeaders) SubtaskCurrentAttemptDetailsHeaders(org.apache.flink.runtime.rest.messages.job.SubtaskCurrentAttemptDetailsHeaders) JobVertexMetricsHandler(org.apache.flink.runtime.rest.handler.job.metrics.JobVertexMetricsHandler) JobsOverviewHeaders(org.apache.flink.runtime.rest.messages.JobsOverviewHeaders) SubtasksTimesHeaders(org.apache.flink.runtime.rest.messages.SubtasksTimesHeaders) JsonArchivist(org.apache.flink.runtime.webmonitor.history.JsonArchivist) SavepointHandlers(org.apache.flink.runtime.rest.handler.job.savepoints.SavepointHandlers) Executors(java.util.concurrent.Executors) TaskManagerCustomLogHeaders(org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerCustomLogHeaders) SubtasksTimesHandler(org.apache.flink.runtime.rest.handler.job.SubtasksTimesHandler) Time(org.apache.flink.api.common.time.Time) ResourceManagerGateway(org.apache.flink.runtime.resourcemanager.ResourceManagerGateway) JobIdsWithStatusesOverviewHeaders(org.apache.flink.runtime.rest.messages.JobIdsWithStatusesOverviewHeaders) ArchivedJson(org.apache.flink.runtime.webmonitor.history.ArchivedJson) JobVertexWatermarksHandler(org.apache.flink.runtime.rest.handler.job.metrics.JobVertexWatermarksHandler) TaskManagerLogFileHandler(org.apache.flink.runtime.rest.handler.taskmanager.TaskManagerLogFileHandler) ClusterOverviewHeaders(org.apache.flink.runtime.rest.messages.ClusterOverviewHeaders) SubtaskExecutionAttemptAccumulatorsHandler(org.apache.flink.runtime.rest.handler.job.SubtaskExecutionAttemptAccumulatorsHandler) JobVertexAccumulatorsHeaders(org.apache.flink.runtime.rest.messages.JobVertexAccumulatorsHeaders) JobManagerCustomLogHandler(org.apache.flink.runtime.rest.handler.cluster.JobManagerCustomLogHandler) SavepointDisposalHandlers(org.apache.flink.runtime.rest.handler.job.savepoints.SavepointDisposalHandlers) ArrayList(java.util.ArrayList) DashboardConfigurationHeaders(org.apache.flink.runtime.rest.messages.DashboardConfigurationHeaders) FutureUtils(org.apache.flink.util.concurrent.FutureUtils) JobManagerMetricsHandler(org.apache.flink.runtime.rest.handler.job.metrics.JobManagerMetricsHandler) ClusterDataSetListHandler(org.apache.flink.runtime.rest.handler.dataset.ClusterDataSetListHandler) JobVertexTaskManagersHandler(org.apache.flink.runtime.rest.handler.job.JobVertexTaskManagersHandler) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) RestOptions(org.apache.flink.configuration.RestOptions) SubtaskExecutionAttemptAccumulatorsHeaders(org.apache.flink.runtime.rest.messages.job.SubtaskExecutionAttemptAccumulatorsHeaders) MetricFetcher(org.apache.flink.runtime.rest.handler.legacy.metrics.MetricFetcher) AbstractRestHandler(org.apache.flink.runtime.rest.handler.AbstractRestHandler) Nullable(javax.annotation.Nullable) JobExecutionResultHandler(org.apache.flink.runtime.rest.handler.job.JobExecutionResultHandler) AggregatingJobsMetricsHandler(org.apache.flink.runtime.rest.handler.job.metrics.AggregatingJobsMetricsHandler) JobVertexAccumulatorsHandler(org.apache.flink.runtime.rest.handler.job.JobVertexAccumulatorsHandler) TaskManagerMetricsHandler(org.apache.flink.runtime.rest.handler.job.metrics.TaskManagerMetricsHandler) JobCancellationHandler(org.apache.flink.runtime.rest.handler.job.JobCancellationHandler) IOException(java.io.IOException) JobVertexFlameGraphHandler(org.apache.flink.runtime.rest.handler.job.JobVertexFlameGraphHandler) JobManagerLogFileHeader(org.apache.flink.runtime.rest.messages.cluster.JobManagerLogFileHeader) File(java.io.File) JobVertexDetailsHandler(org.apache.flink.runtime.rest.handler.job.JobVertexDetailsHandler) JobExceptionsHandler(org.apache.flink.runtime.rest.handler.job.JobExceptionsHandler) TerminationModeQueryParameter(org.apache.flink.runtime.rest.messages.TerminationModeQueryParameter) TaskManagerLogsHeaders(org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerLogsHeaders) JobDetailsHandler(org.apache.flink.runtime.rest.handler.job.JobDetailsHandler) JobVertexThreadInfoTrackerBuilder(org.apache.flink.runtime.webmonitor.threadinfo.JobVertexThreadInfoTrackerBuilder) JobManagerThreadDumpHeaders(org.apache.flink.runtime.rest.messages.cluster.JobManagerThreadDumpHeaders) TaskManagerThreadDumpHeaders(org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerThreadDumpHeaders) TaskManagerStdoutFileHeaders(org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerStdoutFileHeaders) ScheduledFuture(java.util.concurrent.ScheduledFuture) JobVertexBackPressureHeaders(org.apache.flink.runtime.rest.messages.JobVertexBackPressureHeaders) JobVertexThreadInfoStats(org.apache.flink.runtime.webmonitor.threadinfo.JobVertexThreadInfoStats) FileUtils(org.apache.flink.util.FileUtils) JobIdsHandler(org.apache.flink.runtime.rest.handler.job.JobIdsHandler) ExceptionUtils(org.apache.flink.util.ExceptionUtils) TaskManagerLogListHandler(org.apache.flink.runtime.rest.handler.taskmanager.TaskManagerLogListHandler) SubtasksAllAccumulatorsHandler(org.apache.flink.runtime.rest.handler.job.SubtasksAllAccumulatorsHandler) SubtaskExecutionAttemptDetailsHandler(org.apache.flink.runtime.rest.handler.job.SubtaskExecutionAttemptDetailsHandler) JobMetricsHandler(org.apache.flink.runtime.rest.handler.job.metrics.JobMetricsHandler) WebContentHandlerSpecification(org.apache.flink.runtime.rest.handler.legacy.files.WebContentHandlerSpecification) CheckpointStatisticDetailsHeaders(org.apache.flink.runtime.rest.messages.checkpoints.CheckpointStatisticDetailsHeaders) AkkaOptions(org.apache.flink.configuration.AkkaOptions) ExecutorThreadFactory(org.apache.flink.util.concurrent.ExecutorThreadFactory) CheckpointConfigHandler(org.apache.flink.runtime.rest.handler.job.checkpoints.CheckpointConfigHandler) TaskManagerCustomLogHandler(org.apache.flink.runtime.rest.handler.taskmanager.TaskManagerCustomLogHandler) Collection(java.util.Collection) GatewayRetriever(org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever) RestHandlerConfiguration(org.apache.flink.runtime.rest.handler.RestHandlerConfiguration) TransientBlobService(org.apache.flink.runtime.blob.TransientBlobService) UUID(java.util.UUID) JobManagerStdoutFileHeader(org.apache.flink.runtime.rest.messages.cluster.JobManagerStdoutFileHeader) Preconditions(org.apache.flink.util.Preconditions) CheckpointingOptions(org.apache.flink.configuration.CheckpointingOptions) ExecutorUtils(org.apache.flink.util.ExecutorUtils) JobsOverviewHandler(org.apache.flink.runtime.rest.handler.job.JobsOverviewHandler) List(java.util.List) SubtaskCurrentAttemptDetailsHandler(org.apache.flink.runtime.rest.handler.job.SubtaskCurrentAttemptDetailsHandler) CheckpointingStatisticsHandler(org.apache.flink.runtime.rest.handler.job.checkpoints.CheckpointingStatisticsHandler) JobManagerLogListHandler(org.apache.flink.runtime.rest.handler.cluster.JobManagerLogListHandler) JobVertexTaskManagersHeaders(org.apache.flink.runtime.rest.messages.JobVertexTaskManagersHeaders) ClusterDataSetDeleteHandlers(org.apache.flink.runtime.rest.handler.dataset.ClusterDataSetDeleteHandlers) ClientCoordinationHandler(org.apache.flink.runtime.rest.handler.job.coordination.ClientCoordinationHandler) Optional(java.util.Optional) DashboardConfigHandler(org.apache.flink.runtime.rest.handler.cluster.DashboardConfigHandler) SubtaskExecutionAttemptDetailsHeaders(org.apache.flink.runtime.rest.messages.job.SubtaskExecutionAttemptDetailsHeaders) CheckpointStatsCache(org.apache.flink.runtime.rest.handler.job.checkpoints.CheckpointStatsCache) CheckpointConfigHeaders(org.apache.flink.runtime.rest.messages.checkpoints.CheckpointConfigHeaders) JobVertexThreadInfoTracker(org.apache.flink.runtime.webmonitor.threadinfo.JobVertexThreadInfoTracker) LeaderElectionService(org.apache.flink.runtime.leaderelection.LeaderElectionService) ShutdownHandler(org.apache.flink.runtime.rest.handler.cluster.ShutdownHandler) JobVertexDetailsHeaders(org.apache.flink.runtime.rest.messages.JobVertexDetailsHeaders) ConfigurationException(org.apache.flink.util.ConfigurationException) RescalingHandlers(org.apache.flink.runtime.rest.handler.job.rescaling.RescalingHandlers) JobManagerLogFileHandler(org.apache.flink.runtime.rest.handler.cluster.JobManagerLogFileHandler) CompletableFuture(java.util.concurrent.CompletableFuture) JobManagerCustomLogHeaders(org.apache.flink.runtime.rest.messages.cluster.JobManagerCustomLogHeaders) Function(java.util.function.Function) TaskManagerDetailsHandler(org.apache.flink.runtime.rest.handler.taskmanager.TaskManagerDetailsHandler) JobCancellationHeaders(org.apache.flink.runtime.rest.messages.JobCancellationHeaders) LeaderContender(org.apache.flink.runtime.leaderelection.LeaderContender) JobVertexBackPressureHandler(org.apache.flink.runtime.rest.handler.job.JobVertexBackPressureHandler) YarnStopJobTerminationHeaders(org.apache.flink.runtime.rest.messages.YarnStopJobTerminationHeaders) TaskCheckpointStatisticDetailsHandler(org.apache.flink.runtime.rest.handler.job.checkpoints.TaskCheckpointStatisticDetailsHandler) TaskManagersHeaders(org.apache.flink.runtime.rest.messages.taskmanager.TaskManagersHeaders) FatalErrorHandler(org.apache.flink.runtime.rpc.FatalErrorHandler) JobDetailsHeaders(org.apache.flink.runtime.rest.messages.job.JobDetailsHeaders) SubtasksAllAccumulatorsHeaders(org.apache.flink.runtime.rest.messages.SubtasksAllAccumulatorsHeaders) AggregatingSubtasksMetricsHandler(org.apache.flink.runtime.rest.handler.job.metrics.AggregatingSubtasksMetricsHandler) RestServerEndpoint(org.apache.flink.runtime.rest.RestServerEndpoint) JobManagerThreadDumpHandler(org.apache.flink.runtime.rest.handler.cluster.JobManagerThreadDumpHandler) AggregatingTaskManagersMetricsHandler(org.apache.flink.runtime.rest.handler.job.metrics.AggregatingTaskManagersMetricsHandler) ChannelInboundHandler(org.apache.flink.shaded.netty4.io.netty.channel.ChannelInboundHandler) Configuration(org.apache.flink.configuration.Configuration) CheckpointingStatisticsHeaders(org.apache.flink.runtime.rest.messages.checkpoints.CheckpointingStatisticsHeaders) TaskManagersHandler(org.apache.flink.runtime.rest.handler.taskmanager.TaskManagersHandler) YarnCancelJobTerminationHeaders(org.apache.flink.runtime.rest.messages.YarnCancelJobTerminationHeaders) ExecutionGraphInfo(org.apache.flink.runtime.scheduler.ExecutionGraphInfo) ClusterConfigHandler(org.apache.flink.runtime.rest.handler.cluster.ClusterConfigHandler) TimeUnit(java.util.concurrent.TimeUnit) StaticFileServerHandler(org.apache.flink.runtime.rest.handler.legacy.files.StaticFileServerHandler) JobPlanHandler(org.apache.flink.runtime.rest.handler.job.JobPlanHandler) RestHandlerSpecification(org.apache.flink.runtime.rest.handler.RestHandlerSpecification) JobAccumulatorsHandler(org.apache.flink.runtime.rest.handler.job.JobAccumulatorsHandler) ClusterConfigurationInfoHeaders(org.apache.flink.runtime.rest.messages.ClusterConfigurationInfoHeaders) Collections(java.util.Collections) JobConfigHeaders(org.apache.flink.runtime.rest.messages.JobConfigHeaders) JobExceptionsHandler(org.apache.flink.runtime.rest.handler.job.JobExceptionsHandler) ArrayList(java.util.ArrayList) SubtasksAllAccumulatorsHandler(org.apache.flink.runtime.rest.handler.job.SubtasksAllAccumulatorsHandler) AggregatingTaskManagersMetricsHandler(org.apache.flink.runtime.rest.handler.job.metrics.AggregatingTaskManagersMetricsHandler) JobPlanHandler(org.apache.flink.runtime.rest.handler.job.JobPlanHandler) ShutdownHandler(org.apache.flink.runtime.rest.handler.cluster.ShutdownHandler) JobManagerCustomLogHandler(org.apache.flink.runtime.rest.handler.cluster.JobManagerCustomLogHandler) RescalingHandlers(org.apache.flink.runtime.rest.handler.job.rescaling.RescalingHandlers) TaskManagerStdoutFileHandler(org.apache.flink.runtime.rest.handler.taskmanager.TaskManagerStdoutFileHandler) ClusterDataSetListHandler(org.apache.flink.runtime.rest.handler.dataset.ClusterDataSetListHandler) DashboardConfigHandler(org.apache.flink.runtime.rest.handler.cluster.DashboardConfigHandler) JobVertexDetailsHandler(org.apache.flink.runtime.rest.handler.job.JobVertexDetailsHandler) JobVertexTaskManagersHandler(org.apache.flink.runtime.rest.handler.job.JobVertexTaskManagersHandler) TaskManagersHandler(org.apache.flink.runtime.rest.handler.taskmanager.TaskManagersHandler) JobIdsHandler(org.apache.flink.runtime.rest.handler.job.JobIdsHandler) SubtasksTimesHandler(org.apache.flink.runtime.rest.handler.job.SubtasksTimesHandler) TaskManagerLogListHandler(org.apache.flink.runtime.rest.handler.taskmanager.TaskManagerLogListHandler) TaskCheckpointStatisticDetailsHandler(org.apache.flink.runtime.rest.handler.job.checkpoints.TaskCheckpointStatisticDetailsHandler) Duration(java.time.Duration) SubtaskExecutionAttemptAccumulatorsHandler(org.apache.flink.runtime.rest.handler.job.SubtaskExecutionAttemptAccumulatorsHandler) JobVertexBackPressureHandler(org.apache.flink.runtime.rest.handler.job.JobVertexBackPressureHandler) JobManagerLogFileHandler(org.apache.flink.runtime.rest.handler.cluster.JobManagerLogFileHandler) CheckpointingStatisticsHandler(org.apache.flink.runtime.rest.handler.job.checkpoints.CheckpointingStatisticsHandler) SavepointHandlers(org.apache.flink.runtime.rest.handler.job.savepoints.SavepointHandlers) ClusterDataSetDeleteHandlers(org.apache.flink.runtime.rest.handler.dataset.ClusterDataSetDeleteHandlers) TaskManagerThreadDumpHandler(org.apache.flink.runtime.rest.handler.taskmanager.TaskManagerThreadDumpHandler) JobConfigHandler(org.apache.flink.runtime.rest.handler.job.JobConfigHandler) AggregatingSubtasksMetricsHandler(org.apache.flink.runtime.rest.handler.job.metrics.AggregatingSubtasksMetricsHandler) TaskManagerLogFileHandler(org.apache.flink.runtime.rest.handler.taskmanager.TaskManagerLogFileHandler) JobVertexFlameGraphHandler(org.apache.flink.runtime.rest.handler.job.JobVertexFlameGraphHandler) JobsOverviewHandler(org.apache.flink.runtime.rest.handler.job.JobsOverviewHandler) File(java.io.File) TaskManagerCustomLogHandler(org.apache.flink.runtime.rest.handler.taskmanager.TaskManagerCustomLogHandler) SubtaskCurrentAttemptDetailsHandler(org.apache.flink.runtime.rest.handler.job.SubtaskCurrentAttemptDetailsHandler) ClusterConfigHandler(org.apache.flink.runtime.rest.handler.cluster.ClusterConfigHandler) JobVertexMetricsHandler(org.apache.flink.runtime.rest.handler.job.metrics.JobVertexMetricsHandler) ClusterOverviewHandler(org.apache.flink.runtime.rest.handler.cluster.ClusterOverviewHandler) JobVertexWatermarksHandler(org.apache.flink.runtime.rest.handler.job.metrics.JobVertexWatermarksHandler) Time(org.apache.flink.api.common.time.Time) SubtaskExecutionAttemptDetailsHandler(org.apache.flink.runtime.rest.handler.job.SubtaskExecutionAttemptDetailsHandler) SavepointDisposalHandlers(org.apache.flink.runtime.rest.handler.job.savepoints.SavepointDisposalHandlers) JobMetricsHandler(org.apache.flink.runtime.rest.handler.job.metrics.JobMetricsHandler) StaticFileServerHandler(org.apache.flink.runtime.rest.handler.legacy.files.StaticFileServerHandler) JobManagerLogListHandler(org.apache.flink.runtime.rest.handler.cluster.JobManagerLogListHandler) AggregatingJobsMetricsHandler(org.apache.flink.runtime.rest.handler.job.metrics.AggregatingJobsMetricsHandler) ClientCoordinationHandler(org.apache.flink.runtime.rest.handler.job.coordination.ClientCoordinationHandler) TaskManagerDetailsHandler(org.apache.flink.runtime.rest.handler.taskmanager.TaskManagerDetailsHandler) JobVertexTaskManagersHandler(org.apache.flink.runtime.rest.handler.job.JobVertexTaskManagersHandler) JobDetailsHandler(org.apache.flink.runtime.rest.handler.job.JobDetailsHandler) JobManagerMetricsHandler(org.apache.flink.runtime.rest.handler.job.metrics.JobManagerMetricsHandler) CheckpointStatisticDetailsHandler(org.apache.flink.runtime.rest.handler.job.checkpoints.CheckpointStatisticDetailsHandler) TaskCheckpointStatisticDetailsHandler(org.apache.flink.runtime.rest.handler.job.checkpoints.TaskCheckpointStatisticDetailsHandler) TaskManagerMetricsHandler(org.apache.flink.runtime.rest.handler.job.metrics.TaskManagerMetricsHandler) IOException(java.io.IOException) CheckpointConfigHandler(org.apache.flink.runtime.rest.handler.job.checkpoints.CheckpointConfigHandler) JobAccumulatorsHandler(org.apache.flink.runtime.rest.handler.job.JobAccumulatorsHandler) JsonArchivist(org.apache.flink.runtime.webmonitor.history.JsonArchivist) Tuple2(org.apache.flink.api.java.tuple.Tuple2) SubtaskMetricsHandler(org.apache.flink.runtime.rest.handler.job.metrics.SubtaskMetricsHandler) JobCancellationHandler(org.apache.flink.runtime.rest.handler.job.JobCancellationHandler) JobExecutionResultHandler(org.apache.flink.runtime.rest.handler.job.JobExecutionResultHandler) JobVertexAccumulatorsHandler(org.apache.flink.runtime.rest.handler.job.JobVertexAccumulatorsHandler)

Aggregations

IOException (java.io.IOException)2 TransientBlobService (org.apache.flink.runtime.blob.TransientBlobService)2 File (java.io.File)1 Duration (java.time.Duration)1 ArrayList (java.util.ArrayList)1 Collection (java.util.Collection)1 Collections (java.util.Collections)1 List (java.util.List)1 Optional (java.util.Optional)1 UUID (java.util.UUID)1 CompletableFuture (java.util.concurrent.CompletableFuture)1 Executors (java.util.concurrent.Executors)1 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)1 ScheduledFuture (java.util.concurrent.ScheduledFuture)1 TimeUnit (java.util.concurrent.TimeUnit)1 Function (java.util.function.Function)1 Nullable (javax.annotation.Nullable)1 Time (org.apache.flink.api.common.time.Time)1 Tuple2 (org.apache.flink.api.java.tuple.Tuple2)1 AkkaOptions (org.apache.flink.configuration.AkkaOptions)1