Search in sources :

Example 21 with BlobServer

use of org.apache.flink.runtime.blob.BlobServer in project flink by apache.

the class MiniCluster method setupDispatcherResourceManagerComponents.

@GuardedBy("lock")
private void setupDispatcherResourceManagerComponents(Configuration configuration, RpcServiceFactory dispatcherResourceManagerComponentRpcServiceFactory, MetricQueryServiceRetriever metricQueryServiceRetriever) throws Exception {
    dispatcherResourceManagerComponents.addAll(createDispatcherResourceManagerComponents(configuration, dispatcherResourceManagerComponentRpcServiceFactory, haServices, blobServer, heartbeatServices, metricRegistry, metricQueryServiceRetriever, new ShutDownFatalErrorHandler()));
    final Collection<CompletableFuture<ApplicationStatus>> shutDownFutures = new ArrayList<>(dispatcherResourceManagerComponents.size());
    for (DispatcherResourceManagerComponent dispatcherResourceManagerComponent : dispatcherResourceManagerComponents) {
        final CompletableFuture<ApplicationStatus> shutDownFuture = dispatcherResourceManagerComponent.getShutDownFuture();
        FutureUtils.assertNoException(shutDownFuture.thenCompose(applicationStatus -> dispatcherResourceManagerComponent.stopApplication(applicationStatus, null)));
        shutDownFutures.add(shutDownFuture);
    }
    FutureUtils.completeAll(shutDownFutures).whenComplete((ignored, exception) -> closeAsync());
}
Also used : InetAddress(java.net.InetAddress) ClusterOverview(org.apache.flink.runtime.messages.webmonitor.ClusterOverview) MetricRegistry(org.apache.flink.runtime.metrics.MetricRegistry) FunctionUtils(org.apache.flink.util.function.FunctionUtils) Duration(java.time.Duration) JobStatusMessage(org.apache.flink.runtime.client.JobStatusMessage) HighAvailabilityServicesUtils(org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils) GuardedBy(javax.annotation.concurrent.GuardedBy) ClientUtils(org.apache.flink.runtime.client.ClientUtils) Executors(java.util.concurrent.Executors) AccessExecutionGraph(org.apache.flink.runtime.executiongraph.AccessExecutionGraph) HeartbeatServices(org.apache.flink.runtime.heartbeat.HeartbeatServices) ClusterEntrypointUtils(org.apache.flink.runtime.entrypoint.ClusterEntrypointUtils) Time(org.apache.flink.api.common.time.Time) ResourceManagerGateway(org.apache.flink.runtime.resourcemanager.ResourceManagerGateway) FlinkException(org.apache.flink.util.FlinkException) BlobServer(org.apache.flink.runtime.blob.BlobServer) CoordinationResponse(org.apache.flink.runtime.operators.coordination.CoordinationResponse) MetricRegistryImpl(org.apache.flink.runtime.metrics.MetricRegistryImpl) SavepointConfigOptions(org.apache.flink.runtime.jobgraph.SavepointConfigOptions) JobStatus(org.apache.flink.api.common.JobStatus) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) RpcService(org.apache.flink.runtime.rpc.RpcService) ResourceOverview(org.apache.flink.runtime.resourcemanager.ResourceOverview) FutureUtils(org.apache.flink.util.concurrent.FutureUtils) JobExecutionException(org.apache.flink.runtime.client.JobExecutionException) FileOutputFormat(org.apache.flink.api.common.io.FileOutputFormat) Executor(java.util.concurrent.Executor) ApplicationStatus(org.apache.flink.runtime.clusterframework.ApplicationStatus) TaskExecutor(org.apache.flink.runtime.taskexecutor.TaskExecutor) IOException(java.io.IOException) VisibleForTesting(org.apache.flink.annotation.VisibleForTesting) ExecutionException(java.util.concurrent.ExecutionException) JobID(org.apache.flink.api.common.JobID) ConfigurationUtils(org.apache.flink.configuration.ConfigurationUtils) SavepointRestoreSettings(org.apache.flink.runtime.jobgraph.SavepointRestoreSettings) ProcessMetricGroup(org.apache.flink.runtime.metrics.groups.ProcessMetricGroup) ClusterOptions(org.apache.flink.configuration.ClusterOptions) EmbeddedHaServicesWithLeadershipControl(org.apache.flink.runtime.highavailability.nonha.embedded.EmbeddedHaServicesWithLeadershipControl) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) LoggerFactory(org.slf4j.LoggerFactory) ExceptionUtils(org.apache.flink.util.ExceptionUtils) ReporterSetup(org.apache.flink.runtime.metrics.ReporterSetup) InstantiationUtil(org.apache.flink.util.InstantiationUtil) URI(java.net.URI) Preconditions.checkNotNull(org.apache.flink.util.Preconditions.checkNotNull) TaskManagerRunner(org.apache.flink.runtime.taskexecutor.TaskManagerRunner) TriggerSavepointMode(org.apache.flink.runtime.dispatcher.TriggerSavepointMode) HighAvailabilityServices(org.apache.flink.runtime.highavailability.HighAvailabilityServices) LeaderRetriever(org.apache.flink.runtime.webmonitor.retriever.LeaderRetriever) ExecutorThreadFactory(org.apache.flink.util.concurrent.ExecutorThreadFactory) Collection(java.util.Collection) ResourceManagerId(org.apache.flink.runtime.resourcemanager.ResourceManagerId) CompletionException(java.util.concurrent.CompletionException) MetricUtils(org.apache.flink.runtime.metrics.util.MetricUtils) UUID(java.util.UUID) InetSocketAddress(java.net.InetSocketAddress) Collectors(java.util.stream.Collectors) Acknowledge(org.apache.flink.runtime.messages.Acknowledge) ExecutorUtils(org.apache.flink.util.ExecutorUtils) JobExecutionResult(org.apache.flink.api.common.JobExecutionResult) List(java.util.List) SerializedValue(org.apache.flink.util.SerializedValue) CoordinationRequest(org.apache.flink.runtime.operators.coordination.CoordinationRequest) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) Optional(java.util.Optional) DispatcherId(org.apache.flink.runtime.dispatcher.DispatcherId) IllegalConfigurationException(org.apache.flink.configuration.IllegalConfigurationException) HaLeadershipControl(org.apache.flink.runtime.highavailability.nonha.embedded.HaLeadershipControl) SavepointFormatType(org.apache.flink.core.execution.SavepointFormatType) RpcMetricQueryServiceRetriever(org.apache.flink.runtime.webmonitor.retriever.impl.RpcMetricQueryServiceRetriever) CompletableFuture(java.util.concurrent.CompletableFuture) DispatcherGateway(org.apache.flink.runtime.dispatcher.DispatcherGateway) Function(java.util.function.Function) MetricRegistryConfiguration(org.apache.flink.runtime.metrics.MetricRegistryConfiguration) RestoreMode(org.apache.flink.runtime.jobgraph.RestoreMode) JobResult(org.apache.flink.runtime.jobmaster.JobResult) DefaultDispatcherResourceManagerComponentFactory(org.apache.flink.runtime.entrypoint.component.DefaultDispatcherResourceManagerComponentFactory) LeaderRetrievalService(org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService) ExternalResourceInfoProvider(org.apache.flink.runtime.externalresource.ExternalResourceInfoProvider) ClusterInformation(org.apache.flink.runtime.entrypoint.ClusterInformation) FatalErrorHandler(org.apache.flink.runtime.rpc.FatalErrorHandler) RpcGatewayRetriever(org.apache.flink.runtime.webmonitor.retriever.impl.RpcGatewayRetriever) ResourceID(org.apache.flink.runtime.clusterframework.types.ResourceID) Nonnull(javax.annotation.Nonnull) JobSubmissionResult(org.apache.flink.api.common.JobSubmissionResult) ExecutorService(java.util.concurrent.ExecutorService) MetricQueryServiceRetriever(org.apache.flink.runtime.webmonitor.retriever.MetricQueryServiceRetriever) DispatcherResourceManagerComponentFactory(org.apache.flink.runtime.entrypoint.component.DispatcherResourceManagerComponentFactory) ArchivedExecutionGraph(org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph) Preconditions.checkState(org.apache.flink.util.Preconditions.checkState) Logger(org.slf4j.Logger) AutoCloseableAsync(org.apache.flink.util.AutoCloseableAsync) ExponentialBackoffRetryStrategy(org.apache.flink.util.concurrent.ExponentialBackoffRetryStrategy) Configuration(org.apache.flink.configuration.Configuration) Reference(org.apache.flink.util.Reference) RpcUtils(org.apache.flink.runtime.rpc.RpcUtils) ExecutionGraphInfo(org.apache.flink.runtime.scheduler.ExecutionGraphInfo) TimeUnit(java.util.concurrent.TimeUnit) RpcSystem(org.apache.flink.runtime.rpc.RpcSystem) WorkingDirectory(org.apache.flink.runtime.entrypoint.WorkingDirectory) DispatcherResourceManagerComponent(org.apache.flink.runtime.entrypoint.component.DispatcherResourceManagerComponent) MemoryExecutionGraphInfoStore(org.apache.flink.runtime.dispatcher.MemoryExecutionGraphInfoStore) Internal(org.apache.flink.annotation.Internal) BlobCacheService(org.apache.flink.runtime.blob.BlobCacheService) BlobClient(org.apache.flink.runtime.blob.BlobClient) BlobUtils(org.apache.flink.runtime.blob.BlobUtils) StandaloneResourceManagerFactory(org.apache.flink.runtime.resourcemanager.StandaloneResourceManagerFactory) Collections(java.util.Collections) HighAvailabilityOptions(org.apache.flink.configuration.HighAvailabilityOptions) CompletableFuture(java.util.concurrent.CompletableFuture) DispatcherResourceManagerComponent(org.apache.flink.runtime.entrypoint.component.DispatcherResourceManagerComponent) ApplicationStatus(org.apache.flink.runtime.clusterframework.ApplicationStatus) ArrayList(java.util.ArrayList) GuardedBy(javax.annotation.concurrent.GuardedBy)

Aggregations

BlobServer (org.apache.flink.runtime.blob.BlobServer)21 Configuration (org.apache.flink.configuration.Configuration)18 VoidBlobStore (org.apache.flink.runtime.blob.VoidBlobStore)13 InetSocketAddress (java.net.InetSocketAddress)7 JobID (org.apache.flink.api.common.JobID)7 PermanentBlobKey (org.apache.flink.runtime.blob.PermanentBlobKey)7 Before (org.junit.Before)7 Test (org.junit.Test)7 PermanentBlobCache (org.apache.flink.runtime.blob.PermanentBlobCache)6 File (java.io.File)4 ArrayList (java.util.ArrayList)4 JobGraph (org.apache.flink.runtime.jobgraph.JobGraph)4 IOException (java.io.IOException)3 UUID (java.util.UUID)3 URL (java.net.URL)2 Duration (java.time.Duration)2 Collection (java.util.Collection)2 CompletableFuture (java.util.concurrent.CompletableFuture)2 ExecutorService (java.util.concurrent.ExecutorService)2 Time (org.apache.flink.api.common.time.Time)2