Search in sources :

Example 1 with SavepointTriggerMessageParameters

use of org.apache.flink.runtime.rest.messages.job.savepoints.SavepointTriggerMessageParameters in project flink by apache.

the class RestClusterClient method triggerSavepoint.

private CompletableFuture<String> triggerSavepoint(final JobID jobId, @Nullable final String savepointDirectory, final boolean cancelJob, final SavepointFormatType formatType) {
    final SavepointTriggerHeaders savepointTriggerHeaders = SavepointTriggerHeaders.getInstance();
    final SavepointTriggerMessageParameters savepointTriggerMessageParameters = savepointTriggerHeaders.getUnresolvedMessageParameters();
    savepointTriggerMessageParameters.jobID.resolve(jobId);
    final CompletableFuture<TriggerResponse> responseFuture = sendRequest(savepointTriggerHeaders, savepointTriggerMessageParameters, new SavepointTriggerRequestBody(savepointDirectory, cancelJob, formatType, null));
    return responseFuture.thenCompose(savepointTriggerResponseBody -> {
        final TriggerId savepointTriggerId = savepointTriggerResponseBody.getTriggerId();
        return pollSavepointAsync(jobId, savepointTriggerId);
    }).thenApply(savepointInfo -> {
        if (savepointInfo.getFailureCause() != null) {
            throw new CompletionException(savepointInfo.getFailureCause());
        }
        return savepointInfo.getLocation();
    });
}
Also used : JobAccumulatorsHeaders(org.apache.flink.runtime.rest.messages.JobAccumulatorsHeaders) JobSubmissionException(org.apache.flink.runtime.client.JobSubmissionException) Tuple2(org.apache.flink.api.java.tuple.Tuple2) JobAccumulatorsInfo(org.apache.flink.runtime.rest.messages.JobAccumulatorsInfo) SavepointDisposalTriggerHeaders(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalTriggerHeaders) AsynchronousOperationInfo(org.apache.flink.runtime.rest.handler.async.AsynchronousOperationInfo) DefaultClientHighAvailabilityServicesFactory(org.apache.flink.runtime.highavailability.DefaultClientHighAvailabilityServicesFactory) EmptyMessageParameters(org.apache.flink.runtime.rest.messages.EmptyMessageParameters) EmptyRequestBody(org.apache.flink.runtime.rest.messages.EmptyRequestBody) ExponentialWaitStrategy(org.apache.flink.client.program.rest.retry.ExponentialWaitStrategy) JobSubmitHeaders(org.apache.flink.runtime.rest.messages.job.JobSubmitHeaders) Path(org.apache.flink.core.fs.Path) Map(java.util.Map) ShutdownHeaders(org.apache.flink.runtime.rest.messages.cluster.ShutdownHeaders) ClientCoordinationHeaders(org.apache.flink.runtime.rest.messages.job.coordination.ClientCoordinationHeaders) JobStatusMessage(org.apache.flink.runtime.client.JobStatusMessage) JobExecutionResultHeaders(org.apache.flink.runtime.rest.messages.job.JobExecutionResultHeaders) RestConstants(org.apache.flink.runtime.rest.util.RestConstants) JobsOverviewHeaders(org.apache.flink.runtime.rest.messages.JobsOverviewHeaders) Executors(java.util.concurrent.Executors) ConnectTimeoutException(org.apache.flink.shaded.netty4.io.netty.channel.ConnectTimeoutException) JobMessageParameters(org.apache.flink.runtime.rest.messages.JobMessageParameters) ClusterClient(org.apache.flink.client.program.ClusterClient) Time(org.apache.flink.api.common.time.Time) JobSubmitResponseBody(org.apache.flink.runtime.rest.messages.job.JobSubmitResponseBody) FlinkException(org.apache.flink.util.FlinkException) CoordinationResponse(org.apache.flink.runtime.operators.coordination.CoordinationResponse) JobStatus(org.apache.flink.api.common.JobStatus) AccumulatorHelper(org.apache.flink.api.common.accumulators.AccumulatorHelper) Supplier(java.util.function.Supplier) HttpResponseStatus(org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus) ArrayList(java.util.ArrayList) FutureUtils(org.apache.flink.util.concurrent.FutureUtils) StopWithSavepointRequestBody(org.apache.flink.runtime.rest.messages.job.savepoints.stop.StopWithSavepointRequestBody) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) BiConsumer(java.util.function.BiConsumer) ObjectOutputStream(java.io.ObjectOutputStream) SavepointStatusMessageParameters(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointStatusMessageParameters) CheckedSupplier(org.apache.flink.util.function.CheckedSupplier) Nullable(javax.annotation.Nullable) TriggerResponse(org.apache.flink.runtime.rest.handler.async.TriggerResponse) SavepointDisposalStatusMessageParameters(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalStatusMessageParameters) Files(java.nio.file.Files) ApplicationStatus(org.apache.flink.runtime.clusterframework.ApplicationStatus) JobDetailsInfo(org.apache.flink.runtime.rest.messages.job.JobDetailsInfo) SavepointInfo(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointInfo) IOException(java.io.IOException) VisibleForTesting(org.apache.flink.annotation.VisibleForTesting) ExecutionException(java.util.concurrent.ExecutionException) SavepointTriggerMessageParameters(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointTriggerMessageParameters) JobID(org.apache.flink.api.common.JobID) Paths(java.nio.file.Paths) TerminationModeQueryParameter(org.apache.flink.runtime.rest.messages.TerminationModeQueryParameter) SavepointStatusHeaders(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointStatusHeaders) SavepointDisposalRequest(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalRequest) MessageParameters(org.apache.flink.runtime.rest.messages.MessageParameters) TriggerId(org.apache.flink.runtime.rest.messages.TriggerId) URL(java.net.URL) QueueStatus(org.apache.flink.runtime.rest.messages.queue.QueueStatus) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) LoggerFactory(org.slf4j.LoggerFactory) ExceptionUtils(org.apache.flink.util.ExceptionUtils) ClientCoordinationRequestBody(org.apache.flink.runtime.rest.messages.job.coordination.ClientCoordinationRequestBody) JobAccumulatorsMessageParameters(org.apache.flink.runtime.rest.messages.JobAccumulatorsMessageParameters) SavepointTriggerRequestBody(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointTriggerRequestBody) RequestBody(org.apache.flink.runtime.rest.messages.RequestBody) Preconditions.checkNotNull(org.apache.flink.util.Preconditions.checkNotNull) SavepointDisposalStatusHeaders(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalStatusHeaders) RestClientException(org.apache.flink.runtime.rest.util.RestClientException) JobSubmitRequestBody(org.apache.flink.runtime.rest.messages.job.JobSubmitRequestBody) LeaderRetriever(org.apache.flink.runtime.webmonitor.retriever.LeaderRetriever) ExecutorThreadFactory(org.apache.flink.util.concurrent.ExecutorThreadFactory) Predicate(java.util.function.Predicate) Collection(java.util.Collection) CompletionException(java.util.concurrent.CompletionException) SavepointTriggerHeaders(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointTriggerHeaders) DistributedCache(org.apache.flink.api.common.cache.DistributedCache) Collectors(java.util.stream.Collectors) Acknowledge(org.apache.flink.runtime.messages.Acknowledge) StopWithSavepointTriggerHeaders(org.apache.flink.runtime.rest.messages.job.savepoints.stop.StopWithSavepointTriggerHeaders) ExecutorUtils(org.apache.flink.util.ExecutorUtils) FileUpload(org.apache.flink.runtime.rest.FileUpload) List(java.util.List) SerializedValue(org.apache.flink.util.SerializedValue) CoordinationRequest(org.apache.flink.runtime.operators.coordination.CoordinationRequest) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) SavepointFormatType(org.apache.flink.core.execution.SavepointFormatType) ScheduledExecutorServiceAdapter(org.apache.flink.util.concurrent.ScheduledExecutorServiceAdapter) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) MessageHeaders(org.apache.flink.runtime.rest.messages.MessageHeaders) JobCancellationHeaders(org.apache.flink.runtime.rest.messages.JobCancellationHeaders) JobResult(org.apache.flink.runtime.jobmaster.JobResult) WaitStrategy(org.apache.flink.client.program.rest.retry.WaitStrategy) ClientCoordinationMessageParameters(org.apache.flink.runtime.rest.messages.job.coordination.ClientCoordinationMessageParameters) ClientHighAvailabilityServices(org.apache.flink.runtime.highavailability.ClientHighAvailabilityServices) LeaderRetrievalService(org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService) JobDetailsHeaders(org.apache.flink.runtime.rest.messages.job.JobDetailsHeaders) AsynchronouslyCreatedResource(org.apache.flink.runtime.rest.messages.queue.AsynchronouslyCreatedResource) Nonnull(javax.annotation.Nonnull) ExecutorService(java.util.concurrent.ExecutorService) EmptyResponseBody(org.apache.flink.runtime.rest.messages.EmptyResponseBody) Logger(org.slf4j.Logger) MalformedURLException(java.net.MalformedURLException) Configuration(org.apache.flink.configuration.Configuration) ClientHighAvailabilityServicesFactory(org.apache.flink.runtime.highavailability.ClientHighAvailabilityServicesFactory) TimeUnit(java.util.concurrent.TimeUnit) ResponseBody(org.apache.flink.runtime.rest.messages.ResponseBody) JobCancellationMessageParameters(org.apache.flink.runtime.rest.messages.JobCancellationMessageParameters) Collections(java.util.Collections) RestClient(org.apache.flink.runtime.rest.RestClient) TriggerId(org.apache.flink.runtime.rest.messages.TriggerId) CompletionException(java.util.concurrent.CompletionException) SavepointTriggerMessageParameters(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointTriggerMessageParameters) TriggerResponse(org.apache.flink.runtime.rest.handler.async.TriggerResponse) SavepointTriggerRequestBody(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointTriggerRequestBody) SavepointTriggerHeaders(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointTriggerHeaders) StopWithSavepointTriggerHeaders(org.apache.flink.runtime.rest.messages.job.savepoints.stop.StopWithSavepointTriggerHeaders)

Example 2 with SavepointTriggerMessageParameters

use of org.apache.flink.runtime.rest.messages.job.savepoints.SavepointTriggerMessageParameters in project flink by apache.

the class RestClusterClient method stopWithSavepoint.

@Override
public CompletableFuture<String> stopWithSavepoint(final JobID jobId, final boolean advanceToEndOfTime, @Nullable final String savepointDirectory, final SavepointFormatType formatType) {
    final StopWithSavepointTriggerHeaders stopWithSavepointTriggerHeaders = StopWithSavepointTriggerHeaders.getInstance();
    final SavepointTriggerMessageParameters stopWithSavepointTriggerMessageParameters = stopWithSavepointTriggerHeaders.getUnresolvedMessageParameters();
    stopWithSavepointTriggerMessageParameters.jobID.resolve(jobId);
    final CompletableFuture<TriggerResponse> responseFuture = sendRequest(stopWithSavepointTriggerHeaders, stopWithSavepointTriggerMessageParameters, new StopWithSavepointRequestBody(savepointDirectory, advanceToEndOfTime, formatType, null));
    return responseFuture.thenCompose(savepointTriggerResponseBody -> {
        final TriggerId savepointTriggerId = savepointTriggerResponseBody.getTriggerId();
        return pollSavepointAsync(jobId, savepointTriggerId);
    }).thenApply(savepointInfo -> {
        if (savepointInfo.getFailureCause() != null) {
            throw new CompletionException(savepointInfo.getFailureCause());
        }
        return savepointInfo.getLocation();
    });
}
Also used : JobAccumulatorsHeaders(org.apache.flink.runtime.rest.messages.JobAccumulatorsHeaders) JobSubmissionException(org.apache.flink.runtime.client.JobSubmissionException) Tuple2(org.apache.flink.api.java.tuple.Tuple2) JobAccumulatorsInfo(org.apache.flink.runtime.rest.messages.JobAccumulatorsInfo) SavepointDisposalTriggerHeaders(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalTriggerHeaders) AsynchronousOperationInfo(org.apache.flink.runtime.rest.handler.async.AsynchronousOperationInfo) DefaultClientHighAvailabilityServicesFactory(org.apache.flink.runtime.highavailability.DefaultClientHighAvailabilityServicesFactory) EmptyMessageParameters(org.apache.flink.runtime.rest.messages.EmptyMessageParameters) EmptyRequestBody(org.apache.flink.runtime.rest.messages.EmptyRequestBody) ExponentialWaitStrategy(org.apache.flink.client.program.rest.retry.ExponentialWaitStrategy) JobSubmitHeaders(org.apache.flink.runtime.rest.messages.job.JobSubmitHeaders) Path(org.apache.flink.core.fs.Path) Map(java.util.Map) ShutdownHeaders(org.apache.flink.runtime.rest.messages.cluster.ShutdownHeaders) ClientCoordinationHeaders(org.apache.flink.runtime.rest.messages.job.coordination.ClientCoordinationHeaders) JobStatusMessage(org.apache.flink.runtime.client.JobStatusMessage) JobExecutionResultHeaders(org.apache.flink.runtime.rest.messages.job.JobExecutionResultHeaders) RestConstants(org.apache.flink.runtime.rest.util.RestConstants) JobsOverviewHeaders(org.apache.flink.runtime.rest.messages.JobsOverviewHeaders) Executors(java.util.concurrent.Executors) ConnectTimeoutException(org.apache.flink.shaded.netty4.io.netty.channel.ConnectTimeoutException) JobMessageParameters(org.apache.flink.runtime.rest.messages.JobMessageParameters) ClusterClient(org.apache.flink.client.program.ClusterClient) Time(org.apache.flink.api.common.time.Time) JobSubmitResponseBody(org.apache.flink.runtime.rest.messages.job.JobSubmitResponseBody) FlinkException(org.apache.flink.util.FlinkException) CoordinationResponse(org.apache.flink.runtime.operators.coordination.CoordinationResponse) JobStatus(org.apache.flink.api.common.JobStatus) AccumulatorHelper(org.apache.flink.api.common.accumulators.AccumulatorHelper) Supplier(java.util.function.Supplier) HttpResponseStatus(org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus) ArrayList(java.util.ArrayList) FutureUtils(org.apache.flink.util.concurrent.FutureUtils) StopWithSavepointRequestBody(org.apache.flink.runtime.rest.messages.job.savepoints.stop.StopWithSavepointRequestBody) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) BiConsumer(java.util.function.BiConsumer) ObjectOutputStream(java.io.ObjectOutputStream) SavepointStatusMessageParameters(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointStatusMessageParameters) CheckedSupplier(org.apache.flink.util.function.CheckedSupplier) Nullable(javax.annotation.Nullable) TriggerResponse(org.apache.flink.runtime.rest.handler.async.TriggerResponse) SavepointDisposalStatusMessageParameters(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalStatusMessageParameters) Files(java.nio.file.Files) ApplicationStatus(org.apache.flink.runtime.clusterframework.ApplicationStatus) JobDetailsInfo(org.apache.flink.runtime.rest.messages.job.JobDetailsInfo) SavepointInfo(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointInfo) IOException(java.io.IOException) VisibleForTesting(org.apache.flink.annotation.VisibleForTesting) ExecutionException(java.util.concurrent.ExecutionException) SavepointTriggerMessageParameters(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointTriggerMessageParameters) JobID(org.apache.flink.api.common.JobID) Paths(java.nio.file.Paths) TerminationModeQueryParameter(org.apache.flink.runtime.rest.messages.TerminationModeQueryParameter) SavepointStatusHeaders(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointStatusHeaders) SavepointDisposalRequest(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalRequest) MessageParameters(org.apache.flink.runtime.rest.messages.MessageParameters) TriggerId(org.apache.flink.runtime.rest.messages.TriggerId) URL(java.net.URL) QueueStatus(org.apache.flink.runtime.rest.messages.queue.QueueStatus) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) LoggerFactory(org.slf4j.LoggerFactory) ExceptionUtils(org.apache.flink.util.ExceptionUtils) ClientCoordinationRequestBody(org.apache.flink.runtime.rest.messages.job.coordination.ClientCoordinationRequestBody) JobAccumulatorsMessageParameters(org.apache.flink.runtime.rest.messages.JobAccumulatorsMessageParameters) SavepointTriggerRequestBody(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointTriggerRequestBody) RequestBody(org.apache.flink.runtime.rest.messages.RequestBody) Preconditions.checkNotNull(org.apache.flink.util.Preconditions.checkNotNull) SavepointDisposalStatusHeaders(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalStatusHeaders) RestClientException(org.apache.flink.runtime.rest.util.RestClientException) JobSubmitRequestBody(org.apache.flink.runtime.rest.messages.job.JobSubmitRequestBody) LeaderRetriever(org.apache.flink.runtime.webmonitor.retriever.LeaderRetriever) ExecutorThreadFactory(org.apache.flink.util.concurrent.ExecutorThreadFactory) Predicate(java.util.function.Predicate) Collection(java.util.Collection) CompletionException(java.util.concurrent.CompletionException) SavepointTriggerHeaders(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointTriggerHeaders) DistributedCache(org.apache.flink.api.common.cache.DistributedCache) Collectors(java.util.stream.Collectors) Acknowledge(org.apache.flink.runtime.messages.Acknowledge) StopWithSavepointTriggerHeaders(org.apache.flink.runtime.rest.messages.job.savepoints.stop.StopWithSavepointTriggerHeaders) ExecutorUtils(org.apache.flink.util.ExecutorUtils) FileUpload(org.apache.flink.runtime.rest.FileUpload) List(java.util.List) SerializedValue(org.apache.flink.util.SerializedValue) CoordinationRequest(org.apache.flink.runtime.operators.coordination.CoordinationRequest) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) SavepointFormatType(org.apache.flink.core.execution.SavepointFormatType) ScheduledExecutorServiceAdapter(org.apache.flink.util.concurrent.ScheduledExecutorServiceAdapter) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) MessageHeaders(org.apache.flink.runtime.rest.messages.MessageHeaders) JobCancellationHeaders(org.apache.flink.runtime.rest.messages.JobCancellationHeaders) JobResult(org.apache.flink.runtime.jobmaster.JobResult) WaitStrategy(org.apache.flink.client.program.rest.retry.WaitStrategy) ClientCoordinationMessageParameters(org.apache.flink.runtime.rest.messages.job.coordination.ClientCoordinationMessageParameters) ClientHighAvailabilityServices(org.apache.flink.runtime.highavailability.ClientHighAvailabilityServices) LeaderRetrievalService(org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService) JobDetailsHeaders(org.apache.flink.runtime.rest.messages.job.JobDetailsHeaders) AsynchronouslyCreatedResource(org.apache.flink.runtime.rest.messages.queue.AsynchronouslyCreatedResource) Nonnull(javax.annotation.Nonnull) ExecutorService(java.util.concurrent.ExecutorService) EmptyResponseBody(org.apache.flink.runtime.rest.messages.EmptyResponseBody) Logger(org.slf4j.Logger) MalformedURLException(java.net.MalformedURLException) Configuration(org.apache.flink.configuration.Configuration) ClientHighAvailabilityServicesFactory(org.apache.flink.runtime.highavailability.ClientHighAvailabilityServicesFactory) TimeUnit(java.util.concurrent.TimeUnit) ResponseBody(org.apache.flink.runtime.rest.messages.ResponseBody) JobCancellationMessageParameters(org.apache.flink.runtime.rest.messages.JobCancellationMessageParameters) Collections(java.util.Collections) RestClient(org.apache.flink.runtime.rest.RestClient) TriggerId(org.apache.flink.runtime.rest.messages.TriggerId) CompletionException(java.util.concurrent.CompletionException) StopWithSavepointTriggerHeaders(org.apache.flink.runtime.rest.messages.job.savepoints.stop.StopWithSavepointTriggerHeaders) SavepointTriggerMessageParameters(org.apache.flink.runtime.rest.messages.job.savepoints.SavepointTriggerMessageParameters) TriggerResponse(org.apache.flink.runtime.rest.handler.async.TriggerResponse) StopWithSavepointRequestBody(org.apache.flink.runtime.rest.messages.job.savepoints.stop.StopWithSavepointRequestBody)

Aggregations

IOException (java.io.IOException)2 ObjectOutputStream (java.io.ObjectOutputStream)2 MalformedURLException (java.net.MalformedURLException)2 URL (java.net.URL)2 Files (java.nio.file.Files)2 Paths (java.nio.file.Paths)2 ArrayList (java.util.ArrayList)2 Collection (java.util.Collection)2 Collections (java.util.Collections)2 List (java.util.List)2 Map (java.util.Map)2 CompletableFuture (java.util.concurrent.CompletableFuture)2 CompletionException (java.util.concurrent.CompletionException)2 ExecutionException (java.util.concurrent.ExecutionException)2 ExecutorService (java.util.concurrent.ExecutorService)2 Executors (java.util.concurrent.Executors)2 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)2 TimeUnit (java.util.concurrent.TimeUnit)2 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)2 BiConsumer (java.util.function.BiConsumer)2