use of org.apache.flink.runtime.blob.TransientBlobService in project flink by apache.
the class TaskExecutor method putTransientBlobStream.
private CompletableFuture<TransientBlobKey> putTransientBlobStream(InputStream inputStream, String fileTag) {
final TransientBlobService transientBlobService = taskExecutorBlobService.getTransientBlobService();
final TransientBlobKey transientBlobKey;
try {
transientBlobKey = transientBlobService.putTransient(inputStream);
} catch (IOException e) {
log.debug("Could not upload file {}.", fileTag, e);
return FutureUtils.completedExceptionally(new FlinkException("Could not upload file " + fileTag + '.', e));
}
return CompletableFuture.completedFuture(transientBlobKey);
}
use of org.apache.flink.runtime.blob.TransientBlobService in project flink by apache.
the class WebMonitorEndpoint method initializeHandlers.
@Override
protected List<Tuple2<RestHandlerSpecification, ChannelInboundHandler>> initializeHandlers(final CompletableFuture<String> localAddressFuture) {
ArrayList<Tuple2<RestHandlerSpecification, ChannelInboundHandler>> handlers = new ArrayList<>(30);
final Collection<Tuple2<RestHandlerSpecification, ChannelInboundHandler>> webSubmissionHandlers = initializeWebSubmissionHandlers(localAddressFuture);
handlers.addAll(webSubmissionHandlers);
final boolean hasWebSubmissionHandlers = !webSubmissionHandlers.isEmpty();
final Duration asyncOperationStoreDuration = clusterConfiguration.get(RestOptions.ASYNC_OPERATION_STORE_DURATION);
final Time timeout = restConfiguration.getTimeout();
ClusterOverviewHandler clusterOverviewHandler = new ClusterOverviewHandler(leaderRetriever, timeout, responseHeaders, ClusterOverviewHeaders.getInstance());
DashboardConfigHandler dashboardConfigHandler = new DashboardConfigHandler(leaderRetriever, timeout, responseHeaders, DashboardConfigurationHeaders.getInstance(), restConfiguration.getRefreshInterval(), hasWebSubmissionHandlers, restConfiguration.isWebCancelEnabled());
JobIdsHandler jobIdsHandler = new JobIdsHandler(leaderRetriever, timeout, responseHeaders, JobIdsWithStatusesOverviewHeaders.getInstance());
JobsOverviewHandler jobsOverviewHandler = new JobsOverviewHandler(leaderRetriever, timeout, responseHeaders, JobsOverviewHeaders.getInstance());
ClusterConfigHandler clusterConfigurationHandler = new ClusterConfigHandler(leaderRetriever, timeout, responseHeaders, ClusterConfigurationInfoHeaders.getInstance(), clusterConfiguration);
JobConfigHandler jobConfigHandler = new JobConfigHandler(leaderRetriever, timeout, responseHeaders, JobConfigHeaders.getInstance(), executionGraphCache, executor);
CheckpointConfigHandler checkpointConfigHandler = new CheckpointConfigHandler(leaderRetriever, timeout, responseHeaders, CheckpointConfigHeaders.getInstance(), executionGraphCache, executor);
CheckpointingStatisticsHandler checkpointStatisticsHandler = new CheckpointingStatisticsHandler(leaderRetriever, timeout, responseHeaders, CheckpointingStatisticsHeaders.getInstance(), executionGraphCache, executor);
CheckpointStatisticDetailsHandler checkpointStatisticDetailsHandler = new CheckpointStatisticDetailsHandler(leaderRetriever, timeout, responseHeaders, CheckpointStatisticDetailsHeaders.getInstance(), executionGraphCache, executor, checkpointStatsCache);
JobPlanHandler jobPlanHandler = new JobPlanHandler(leaderRetriever, timeout, responseHeaders, JobPlanHeaders.getInstance(), executionGraphCache, executor);
TaskCheckpointStatisticDetailsHandler taskCheckpointStatisticDetailsHandler = new TaskCheckpointStatisticDetailsHandler(leaderRetriever, timeout, responseHeaders, TaskCheckpointStatisticsHeaders.getInstance(), executionGraphCache, executor, checkpointStatsCache);
JobExceptionsHandler jobExceptionsHandler = new JobExceptionsHandler(leaderRetriever, timeout, responseHeaders, JobExceptionsHeaders.getInstance(), executionGraphCache, executor);
JobVertexAccumulatorsHandler jobVertexAccumulatorsHandler = new JobVertexAccumulatorsHandler(leaderRetriever, timeout, responseHeaders, JobVertexAccumulatorsHeaders.getInstance(), executionGraphCache, executor);
SubtasksAllAccumulatorsHandler subtasksAllAccumulatorsHandler = new SubtasksAllAccumulatorsHandler(leaderRetriever, timeout, responseHeaders, SubtasksAllAccumulatorsHeaders.getInstance(), executionGraphCache, executor);
TaskManagersHandler taskManagersHandler = new TaskManagersHandler(leaderRetriever, timeout, responseHeaders, TaskManagersHeaders.getInstance(), resourceManagerRetriever);
TaskManagerDetailsHandler taskManagerDetailsHandler = new TaskManagerDetailsHandler(leaderRetriever, timeout, responseHeaders, TaskManagerDetailsHeaders.getInstance(), resourceManagerRetriever, metricFetcher);
final JobDetailsHandler jobDetailsHandler = new JobDetailsHandler(leaderRetriever, timeout, responseHeaders, JobDetailsHeaders.getInstance(), executionGraphCache, executor, metricFetcher);
JobAccumulatorsHandler jobAccumulatorsHandler = new JobAccumulatorsHandler(leaderRetriever, timeout, responseHeaders, JobAccumulatorsHeaders.getInstance(), executionGraphCache, executor);
SubtasksTimesHandler subtasksTimesHandler = new SubtasksTimesHandler(leaderRetriever, timeout, responseHeaders, SubtasksTimesHeaders.getInstance(), executionGraphCache, executor);
final JobVertexMetricsHandler jobVertexMetricsHandler = new JobVertexMetricsHandler(leaderRetriever, timeout, responseHeaders, metricFetcher);
final JobVertexWatermarksHandler jobVertexWatermarksHandler = new JobVertexWatermarksHandler(leaderRetriever, timeout, responseHeaders, metricFetcher, executionGraphCache, executor);
final JobMetricsHandler jobMetricsHandler = new JobMetricsHandler(leaderRetriever, timeout, responseHeaders, metricFetcher);
final SubtaskMetricsHandler subtaskMetricsHandler = new SubtaskMetricsHandler(leaderRetriever, timeout, responseHeaders, metricFetcher);
final TaskManagerMetricsHandler taskManagerMetricsHandler = new TaskManagerMetricsHandler(leaderRetriever, timeout, responseHeaders, metricFetcher);
final JobManagerMetricsHandler jobManagerMetricsHandler = new JobManagerMetricsHandler(leaderRetriever, timeout, responseHeaders, metricFetcher);
final AggregatingTaskManagersMetricsHandler aggregatingTaskManagersMetricsHandler = new AggregatingTaskManagersMetricsHandler(leaderRetriever, timeout, responseHeaders, executor, metricFetcher);
final AggregatingJobsMetricsHandler aggregatingJobsMetricsHandler = new AggregatingJobsMetricsHandler(leaderRetriever, timeout, responseHeaders, executor, metricFetcher);
final AggregatingSubtasksMetricsHandler aggregatingSubtasksMetricsHandler = new AggregatingSubtasksMetricsHandler(leaderRetriever, timeout, responseHeaders, executor, metricFetcher);
final JobVertexTaskManagersHandler jobVertexTaskManagersHandler = new JobVertexTaskManagersHandler(leaderRetriever, timeout, responseHeaders, JobVertexTaskManagersHeaders.getInstance(), executionGraphCache, executor, metricFetcher);
final JobExecutionResultHandler jobExecutionResultHandler = new JobExecutionResultHandler(leaderRetriever, timeout, responseHeaders);
final String defaultSavepointDir = clusterConfiguration.getString(CheckpointingOptions.SAVEPOINT_DIRECTORY);
final SavepointHandlers savepointHandlers = new SavepointHandlers(defaultSavepointDir);
final SavepointHandlers.StopWithSavepointHandler stopWithSavepointHandler = savepointHandlers.new StopWithSavepointHandler(leaderRetriever, timeout, responseHeaders);
final SavepointHandlers.SavepointTriggerHandler savepointTriggerHandler = savepointHandlers.new SavepointTriggerHandler(leaderRetriever, timeout, responseHeaders);
final SavepointHandlers.SavepointStatusHandler savepointStatusHandler = new SavepointHandlers.SavepointStatusHandler(leaderRetriever, timeout, responseHeaders);
final SubtaskExecutionAttemptDetailsHandler subtaskExecutionAttemptDetailsHandler = new SubtaskExecutionAttemptDetailsHandler(leaderRetriever, timeout, responseHeaders, SubtaskExecutionAttemptDetailsHeaders.getInstance(), executionGraphCache, executor, metricFetcher);
final SubtaskExecutionAttemptAccumulatorsHandler subtaskExecutionAttemptAccumulatorsHandler = new SubtaskExecutionAttemptAccumulatorsHandler(leaderRetriever, timeout, responseHeaders, SubtaskExecutionAttemptAccumulatorsHeaders.getInstance(), executionGraphCache, executor);
final SubtaskCurrentAttemptDetailsHandler subtaskCurrentAttemptDetailsHandler = new SubtaskCurrentAttemptDetailsHandler(leaderRetriever, timeout, responseHeaders, SubtaskCurrentAttemptDetailsHeaders.getInstance(), executionGraphCache, executor, metricFetcher);
final RescalingHandlers rescalingHandlers = new RescalingHandlers(asyncOperationStoreDuration);
final RescalingHandlers.RescalingTriggerHandler rescalingTriggerHandler = rescalingHandlers.new RescalingTriggerHandler(leaderRetriever, timeout, responseHeaders);
final RescalingHandlers.RescalingStatusHandler rescalingStatusHandler = rescalingHandlers.new RescalingStatusHandler(leaderRetriever, timeout, responseHeaders);
final JobVertexBackPressureHandler jobVertexBackPressureHandler = new JobVertexBackPressureHandler(leaderRetriever, timeout, responseHeaders, JobVertexBackPressureHeaders.getInstance(), metricFetcher);
final JobCancellationHandler jobCancelTerminationHandler = new JobCancellationHandler(leaderRetriever, timeout, responseHeaders, JobCancellationHeaders.getInstance(), TerminationModeQueryParameter.TerminationMode.CANCEL);
// use a separate handler for the yarn-cancel to ensure close() is only called once
final JobCancellationHandler yarnJobCancelTerminationHandler = new JobCancellationHandler(leaderRetriever, timeout, responseHeaders, JobCancellationHeaders.getInstance(), TerminationModeQueryParameter.TerminationMode.CANCEL);
// this is kept just for legacy reasons. STOP has been replaced by STOP-WITH-SAVEPOINT.
final JobCancellationHandler jobStopTerminationHandler = new JobCancellationHandler(leaderRetriever, timeout, responseHeaders, JobCancellationHeaders.getInstance(), TerminationModeQueryParameter.TerminationMode.STOP);
final JobVertexDetailsHandler jobVertexDetailsHandler = new JobVertexDetailsHandler(leaderRetriever, timeout, responseHeaders, JobVertexDetailsHeaders.getInstance(), executionGraphCache, executor, metricFetcher);
final SavepointDisposalHandlers savepointDisposalHandlers = new SavepointDisposalHandlers(asyncOperationStoreDuration);
final SavepointDisposalHandlers.SavepointDisposalTriggerHandler savepointDisposalTriggerHandler = savepointDisposalHandlers.new SavepointDisposalTriggerHandler(leaderRetriever, timeout, responseHeaders);
final SavepointDisposalHandlers.SavepointDisposalStatusHandler savepointDisposalStatusHandler = savepointDisposalHandlers.new SavepointDisposalStatusHandler(leaderRetriever, timeout, responseHeaders);
final ClusterDataSetListHandler clusterDataSetListHandler = new ClusterDataSetListHandler(leaderRetriever, timeout, responseHeaders, resourceManagerRetriever);
final ClusterDataSetDeleteHandlers clusterDataSetDeleteHandlers = new ClusterDataSetDeleteHandlers(asyncOperationStoreDuration);
final ClusterDataSetDeleteHandlers.ClusterDataSetDeleteTriggerHandler clusterDataSetDeleteTriggerHandler = clusterDataSetDeleteHandlers.new ClusterDataSetDeleteTriggerHandler(leaderRetriever, timeout, responseHeaders, resourceManagerRetriever);
final ClusterDataSetDeleteHandlers.ClusterDataSetDeleteStatusHandler clusterDataSetDeleteStatusHandler = clusterDataSetDeleteHandlers.new ClusterDataSetDeleteStatusHandler(leaderRetriever, timeout, responseHeaders);
final ClientCoordinationHandler clientCoordinationHandler = new ClientCoordinationHandler(leaderRetriever, timeout, responseHeaders, ClientCoordinationHeaders.getInstance());
final ShutdownHandler shutdownHandler = new ShutdownHandler(leaderRetriever, timeout, responseHeaders, ShutdownHeaders.getInstance());
final File webUiDir = restConfiguration.getWebUiDir();
Optional<StaticFileServerHandler<T>> optWebContent;
try {
optWebContent = WebMonitorUtils.tryLoadWebContent(leaderRetriever, timeout, webUiDir);
} catch (IOException e) {
log.warn("Could not load web content handler.", e);
optWebContent = Optional.empty();
}
handlers.add(Tuple2.of(clusterOverviewHandler.getMessageHeaders(), clusterOverviewHandler));
handlers.add(Tuple2.of(clusterConfigurationHandler.getMessageHeaders(), clusterConfigurationHandler));
handlers.add(Tuple2.of(dashboardConfigHandler.getMessageHeaders(), dashboardConfigHandler));
handlers.add(Tuple2.of(jobIdsHandler.getMessageHeaders(), jobIdsHandler));
handlers.add(Tuple2.of(jobsOverviewHandler.getMessageHeaders(), jobsOverviewHandler));
handlers.add(Tuple2.of(jobConfigHandler.getMessageHeaders(), jobConfigHandler));
handlers.add(Tuple2.of(checkpointConfigHandler.getMessageHeaders(), checkpointConfigHandler));
handlers.add(Tuple2.of(checkpointStatisticsHandler.getMessageHeaders(), checkpointStatisticsHandler));
handlers.add(Tuple2.of(checkpointStatisticDetailsHandler.getMessageHeaders(), checkpointStatisticDetailsHandler));
handlers.add(Tuple2.of(jobPlanHandler.getMessageHeaders(), jobPlanHandler));
handlers.add(Tuple2.of(taskCheckpointStatisticDetailsHandler.getMessageHeaders(), taskCheckpointStatisticDetailsHandler));
handlers.add(Tuple2.of(jobExceptionsHandler.getMessageHeaders(), jobExceptionsHandler));
handlers.add(Tuple2.of(jobVertexAccumulatorsHandler.getMessageHeaders(), jobVertexAccumulatorsHandler));
handlers.add(Tuple2.of(subtasksAllAccumulatorsHandler.getMessageHeaders(), subtasksAllAccumulatorsHandler));
handlers.add(Tuple2.of(jobDetailsHandler.getMessageHeaders(), jobDetailsHandler));
handlers.add(Tuple2.of(jobAccumulatorsHandler.getMessageHeaders(), jobAccumulatorsHandler));
handlers.add(Tuple2.of(taskManagersHandler.getMessageHeaders(), taskManagersHandler));
handlers.add(Tuple2.of(taskManagerDetailsHandler.getMessageHeaders(), taskManagerDetailsHandler));
handlers.add(Tuple2.of(subtasksTimesHandler.getMessageHeaders(), subtasksTimesHandler));
handlers.add(Tuple2.of(jobVertexMetricsHandler.getMessageHeaders(), jobVertexMetricsHandler));
handlers.add(Tuple2.of(jobVertexWatermarksHandler.getMessageHeaders(), jobVertexWatermarksHandler));
handlers.add(Tuple2.of(jobMetricsHandler.getMessageHeaders(), jobMetricsHandler));
handlers.add(Tuple2.of(subtaskMetricsHandler.getMessageHeaders(), subtaskMetricsHandler));
handlers.add(Tuple2.of(taskManagerMetricsHandler.getMessageHeaders(), taskManagerMetricsHandler));
handlers.add(Tuple2.of(jobManagerMetricsHandler.getMessageHeaders(), jobManagerMetricsHandler));
handlers.add(Tuple2.of(aggregatingTaskManagersMetricsHandler.getMessageHeaders(), aggregatingTaskManagersMetricsHandler));
handlers.add(Tuple2.of(aggregatingJobsMetricsHandler.getMessageHeaders(), aggregatingJobsMetricsHandler));
handlers.add(Tuple2.of(aggregatingSubtasksMetricsHandler.getMessageHeaders(), aggregatingSubtasksMetricsHandler));
handlers.add(Tuple2.of(jobExecutionResultHandler.getMessageHeaders(), jobExecutionResultHandler));
handlers.add(Tuple2.of(savepointTriggerHandler.getMessageHeaders(), savepointTriggerHandler));
handlers.add(Tuple2.of(stopWithSavepointHandler.getMessageHeaders(), stopWithSavepointHandler));
handlers.add(Tuple2.of(savepointStatusHandler.getMessageHeaders(), savepointStatusHandler));
handlers.add(Tuple2.of(subtaskExecutionAttemptDetailsHandler.getMessageHeaders(), subtaskExecutionAttemptDetailsHandler));
handlers.add(Tuple2.of(subtaskExecutionAttemptAccumulatorsHandler.getMessageHeaders(), subtaskExecutionAttemptAccumulatorsHandler));
handlers.add(Tuple2.of(subtaskCurrentAttemptDetailsHandler.getMessageHeaders(), subtaskCurrentAttemptDetailsHandler));
handlers.add(Tuple2.of(jobVertexTaskManagersHandler.getMessageHeaders(), jobVertexTaskManagersHandler));
handlers.add(Tuple2.of(jobVertexBackPressureHandler.getMessageHeaders(), jobVertexBackPressureHandler));
final AbstractRestHandler<?, ?, ?, ?> jobVertexFlameGraphHandler;
if (clusterConfiguration.get(RestOptions.ENABLE_FLAMEGRAPH)) {
jobVertexFlameGraphHandler = new JobVertexFlameGraphHandler(leaderRetriever, timeout, responseHeaders, executionGraphCache, executor, initializeThreadInfoTracker(executor));
} else {
jobVertexFlameGraphHandler = JobVertexFlameGraphHandler.disabledHandler(leaderRetriever, timeout, responseHeaders);
}
handlers.add(Tuple2.of(jobVertexFlameGraphHandler.getMessageHeaders(), jobVertexFlameGraphHandler));
handlers.add(Tuple2.of(jobCancelTerminationHandler.getMessageHeaders(), jobCancelTerminationHandler));
handlers.add(Tuple2.of(jobVertexDetailsHandler.getMessageHeaders(), jobVertexDetailsHandler));
handlers.add(Tuple2.of(rescalingTriggerHandler.getMessageHeaders(), rescalingTriggerHandler));
handlers.add(Tuple2.of(rescalingStatusHandler.getMessageHeaders(), rescalingStatusHandler));
handlers.add(Tuple2.of(savepointDisposalTriggerHandler.getMessageHeaders(), savepointDisposalTriggerHandler));
handlers.add(Tuple2.of(savepointDisposalStatusHandler.getMessageHeaders(), savepointDisposalStatusHandler));
handlers.add(Tuple2.of(clusterDataSetListHandler.getMessageHeaders(), clusterDataSetListHandler));
handlers.add(Tuple2.of(clusterDataSetDeleteTriggerHandler.getMessageHeaders(), clusterDataSetDeleteTriggerHandler));
handlers.add(Tuple2.of(clusterDataSetDeleteStatusHandler.getMessageHeaders(), clusterDataSetDeleteStatusHandler));
handlers.add(Tuple2.of(clientCoordinationHandler.getMessageHeaders(), clientCoordinationHandler));
// TODO: Remove once the Yarn proxy can forward all REST verbs
handlers.add(Tuple2.of(YarnCancelJobTerminationHeaders.getInstance(), yarnJobCancelTerminationHandler));
handlers.add(Tuple2.of(YarnStopJobTerminationHeaders.getInstance(), jobStopTerminationHandler));
handlers.add(Tuple2.of(shutdownHandler.getMessageHeaders(), shutdownHandler));
optWebContent.ifPresent(webContent -> {
handlers.add(Tuple2.of(WebContentHandlerSpecification.getInstance(), webContent));
hasWebUI = true;
});
// load the log and stdout file handler for the main cluster component
final WebMonitorUtils.LogFileLocation logFileLocation = WebMonitorUtils.LogFileLocation.find(clusterConfiguration);
final JobManagerLogFileHandler jobManagerLogFileHandler = new JobManagerLogFileHandler(leaderRetriever, timeout, responseHeaders, JobManagerLogFileHeader.getInstance(), logFileLocation.logFile);
final JobManagerLogFileHandler jobManagerStdoutFileHandler = new JobManagerLogFileHandler(leaderRetriever, timeout, responseHeaders, JobManagerStdoutFileHeader.getInstance(), logFileLocation.stdOutFile);
final JobManagerCustomLogHandler jobManagerCustomLogHandler = new JobManagerCustomLogHandler(leaderRetriever, timeout, responseHeaders, JobManagerCustomLogHeaders.getInstance(), logFileLocation.logDir);
final JobManagerLogListHandler jobManagerLogListHandler = new JobManagerLogListHandler(leaderRetriever, timeout, responseHeaders, JobManagerLogListHeaders.getInstance(), logFileLocation.logDir);
final JobManagerThreadDumpHandler jobManagerThreadDumpHandler = new JobManagerThreadDumpHandler(leaderRetriever, timeout, responseHeaders, JobManagerThreadDumpHeaders.getInstance());
handlers.add(Tuple2.of(JobManagerLogFileHeader.getInstance(), jobManagerLogFileHandler));
handlers.add(Tuple2.of(JobManagerStdoutFileHeader.getInstance(), jobManagerStdoutFileHandler));
handlers.add(Tuple2.of(JobManagerCustomLogHeaders.getInstance(), jobManagerCustomLogHandler));
handlers.add(Tuple2.of(JobManagerLogListHeaders.getInstance(), jobManagerLogListHandler));
handlers.add(Tuple2.of(JobManagerThreadDumpHeaders.getInstance(), jobManagerThreadDumpHandler));
// TaskManager log and stdout file handler
final Time cacheEntryDuration = Time.milliseconds(restConfiguration.getRefreshInterval());
final TaskManagerLogFileHandler taskManagerLogFileHandler = new TaskManagerLogFileHandler(leaderRetriever, timeout, responseHeaders, TaskManagerLogFileHeaders.getInstance(), resourceManagerRetriever, transientBlobService, cacheEntryDuration);
final TaskManagerStdoutFileHandler taskManagerStdoutFileHandler = new TaskManagerStdoutFileHandler(leaderRetriever, timeout, responseHeaders, TaskManagerStdoutFileHeaders.getInstance(), resourceManagerRetriever, transientBlobService, cacheEntryDuration);
final TaskManagerCustomLogHandler taskManagerCustomLogHandler = new TaskManagerCustomLogHandler(leaderRetriever, timeout, responseHeaders, TaskManagerCustomLogHeaders.getInstance(), resourceManagerRetriever, transientBlobService, cacheEntryDuration);
final TaskManagerLogListHandler taskManagerLogListHandler = new TaskManagerLogListHandler(leaderRetriever, timeout, responseHeaders, TaskManagerLogsHeaders.getInstance(), resourceManagerRetriever);
final TaskManagerThreadDumpHandler taskManagerThreadDumpFileHandler = new TaskManagerThreadDumpHandler(leaderRetriever, timeout, responseHeaders, TaskManagerThreadDumpHeaders.getInstance(), resourceManagerRetriever);
handlers.add(Tuple2.of(TaskManagerLogFileHeaders.getInstance(), taskManagerLogFileHandler));
handlers.add(Tuple2.of(TaskManagerStdoutFileHeaders.getInstance(), taskManagerStdoutFileHandler));
handlers.add(Tuple2.of(TaskManagerCustomLogHeaders.getInstance(), taskManagerCustomLogHandler));
handlers.add(Tuple2.of(TaskManagerLogsHeaders.getInstance(), taskManagerLogListHandler));
handlers.add(Tuple2.of(TaskManagerThreadDumpHeaders.getInstance(), taskManagerThreadDumpFileHandler));
handlers.stream().map(tuple -> tuple.f1).filter(handler -> handler instanceof JsonArchivist).forEachOrdered(handler -> archivingHandlers.add((JsonArchivist) handler));
return handlers;
}
Aggregations