Search in sources :

Example 1 with DetachedJobExecutionResult

use of org.apache.flink.core.execution.DetachedJobExecutionResult in project flink by apache.

the class ContextEnvironment method getJobExecutionResult.

private JobExecutionResult getJobExecutionResult(final JobClient jobClient) throws Exception {
    checkNotNull(jobClient);
    JobExecutionResult jobExecutionResult;
    if (getConfiguration().getBoolean(DeploymentOptions.ATTACHED)) {
        CompletableFuture<JobExecutionResult> jobExecutionResultFuture = jobClient.getJobExecutionResult();
        if (getConfiguration().getBoolean(DeploymentOptions.SHUTDOWN_IF_ATTACHED)) {
            Thread shutdownHook = ShutdownHookUtil.addShutdownHook(() -> {
                // wait a smidgen to allow the async request to go through
                // before
                // the jvm exits
                jobClient.cancel().get(1, TimeUnit.SECONDS);
            }, ContextEnvironment.class.getSimpleName(), LOG);
            jobExecutionResultFuture.whenComplete((ignored, throwable) -> ShutdownHookUtil.removeShutdownHook(shutdownHook, ContextEnvironment.class.getSimpleName(), LOG));
        }
        jobExecutionResult = jobExecutionResultFuture.get();
        if (!suppressSysout) {
            System.out.println(jobExecutionResult);
        }
        LOG.info(String.valueOf(jobExecutionResult));
    } else {
        jobExecutionResult = new DetachedJobExecutionResult(jobClient.getJobID());
    }
    return jobExecutionResult;
}
Also used : DetachedJobExecutionResult(org.apache.flink.core.execution.DetachedJobExecutionResult) JobExecutionResult(org.apache.flink.api.common.JobExecutionResult) DetachedJobExecutionResult(org.apache.flink.core.execution.DetachedJobExecutionResult)

Example 2 with DetachedJobExecutionResult

use of org.apache.flink.core.execution.DetachedJobExecutionResult in project flink by apache.

the class StreamExecutionEnvironment method execute.

/**
 * Triggers the program execution. The environment will execute all parts of the program that
 * have resulted in a "sink" operation. Sink operations are for example printing results or
 * forwarding them to a message queue.
 *
 * @param streamGraph the stream graph representing the transformations
 * @return The result of the job execution, containing elapsed time and accumulators.
 * @throws Exception which occurs during job execution.
 */
@Internal
public JobExecutionResult execute(StreamGraph streamGraph) throws Exception {
    final JobClient jobClient = executeAsync(streamGraph);
    try {
        final JobExecutionResult jobExecutionResult;
        if (configuration.getBoolean(DeploymentOptions.ATTACHED)) {
            jobExecutionResult = jobClient.getJobExecutionResult().get();
        } else {
            jobExecutionResult = new DetachedJobExecutionResult(jobClient.getJobID());
        }
        jobListeners.forEach(jobListener -> jobListener.onJobExecuted(jobExecutionResult, null));
        return jobExecutionResult;
    } catch (Throwable t) {
        // get() on the JobExecutionResult Future will throw an ExecutionException. This
        // behaviour was largely not there in Flink versions before the PipelineExecutor
        // refactoring so we should strip that exception.
        Throwable strippedException = ExceptionUtils.stripExecutionException(t);
        jobListeners.forEach(jobListener -> {
            jobListener.onJobExecuted(null, strippedException);
        });
        ExceptionUtils.rethrowException(strippedException);
        // never reached, only make javac happy
        return null;
    }
}
Also used : Arrays(java.util.Arrays) SocketTextStreamFunction(org.apache.flink.streaming.api.functions.source.SocketTextStreamFunction) Tuple3(org.apache.flink.api.java.tuple.Tuple3) Tuple2(org.apache.flink.api.java.tuple.Tuple2) FromIteratorFunction(org.apache.flink.streaming.api.functions.source.FromIteratorFunction) StreamGraphGenerator(org.apache.flink.streaming.api.graph.StreamGraphGenerator) MemorySize(org.apache.flink.configuration.MemorySize) StateBackend(org.apache.flink.runtime.state.StateBackend) WrappingRuntimeException(org.apache.flink.util.WrappingRuntimeException) SplittableIterator(org.apache.flink.util.SplittableIterator) InvalidProgramException(org.apache.flink.api.common.InvalidProgramException) StateBackendLoader(org.apache.flink.runtime.state.StateBackendLoader) Path(org.apache.flink.core.fs.Path) ResourceSpec(org.apache.flink.api.common.operators.ResourceSpec) Map(java.util.Map) StatefulSequenceSource(org.apache.flink.streaming.api.functions.source.StatefulSequenceSource) StreamGraph(org.apache.flink.streaming.api.graph.StreamGraph) FileMonitoringFunction(org.apache.flink.streaming.api.functions.source.FileMonitoringFunction) DefaultExecutorServiceLoader(org.apache.flink.core.execution.DefaultExecutorServiceLoader) ContinuousFileMonitoringFunction(org.apache.flink.streaming.api.functions.source.ContinuousFileMonitoringFunction) PipelineExecutorFactory(org.apache.flink.core.execution.PipelineExecutorFactory) FromElementsFunction(org.apache.flink.streaming.api.functions.source.FromElementsFunction) TernaryBoolean(org.apache.flink.util.TernaryBoolean) Serializable(java.io.Serializable) TypeExtractor(org.apache.flink.api.java.typeutils.TypeExtractor) ResultTypeQueryable(org.apache.flink.api.java.typeutils.ResultTypeQueryable) Source(org.apache.flink.api.connector.source.Source) Time(org.apache.flink.api.common.time.Time) Boundedness(org.apache.flink.api.connector.source.Boundedness) FlinkException(org.apache.flink.util.FlinkException) NumberSequenceSource(org.apache.flink.api.connector.source.lib.NumberSequenceSource) PipelineExecutorServiceLoader(org.apache.flink.core.execution.PipelineExecutorServiceLoader) SlotSharingGroupUtils(org.apache.flink.api.common.operators.util.SlotSharingGroupUtils) UnmodifiableConfiguration(org.apache.flink.configuration.UnmodifiableConfiguration) ArrayList(java.util.ArrayList) SourceFunction(org.apache.flink.streaming.api.functions.source.SourceFunction) ReadableConfig(org.apache.flink.configuration.ReadableConfig) JobListener(org.apache.flink.core.execution.JobListener) InvalidTypesException(org.apache.flink.api.common.functions.InvalidTypesException) RestOptions(org.apache.flink.configuration.RestOptions) ConfigOption(org.apache.flink.configuration.ConfigOption) Nullable(javax.annotation.Nullable) DetachedJobExecutionResult(org.apache.flink.core.execution.DetachedJobExecutionResult) TimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic) Serializer(com.esotericsoftware.kryo.Serializer) SingleOutputStreamOperator(org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator) MissingTypeInfo(org.apache.flink.api.java.typeutils.MissingTypeInfo) IOException(java.io.IOException) JobClient(org.apache.flink.core.execution.JobClient) DynamicCodeLoadingException(org.apache.flink.util.DynamicCodeLoadingException) ExecutionException(java.util.concurrent.ExecutionException) TextInputFormat(org.apache.flink.api.java.io.TextInputFormat) CoreOptions(org.apache.flink.configuration.CoreOptions) Transformation(org.apache.flink.api.dag.Transformation) PojoTypeInfo(org.apache.flink.api.java.typeutils.PojoTypeInfo) FileProcessingMode(org.apache.flink.streaming.api.functions.source.FileProcessingMode) PipelineOptions(org.apache.flink.configuration.PipelineOptions) CheckpointingMode(org.apache.flink.streaming.api.CheckpointingMode) ExceptionUtils(org.apache.flink.util.ExceptionUtils) SlotSharingGroup(org.apache.flink.api.common.operators.SlotSharingGroup) FileInputFormat(org.apache.flink.api.common.io.FileInputFormat) BasicTypeInfo(org.apache.flink.api.common.typeinfo.BasicTypeInfo) InstantiationUtil(org.apache.flink.util.InstantiationUtil) InputFormat(org.apache.flink.api.common.io.InputFormat) Public(org.apache.flink.annotation.Public) URI(java.net.URI) Preconditions.checkNotNull(org.apache.flink.util.Preconditions.checkNotNull) TypeInformation(org.apache.flink.api.common.typeinfo.TypeInformation) ExecutionOptions(org.apache.flink.configuration.ExecutionOptions) TimestampedFileInputSplit(org.apache.flink.streaming.api.functions.source.TimestampedFileInputSplit) Collection(java.util.Collection) WatermarkStrategy(org.apache.flink.api.common.eventtime.WatermarkStrategy) DistributedCache(org.apache.flink.api.common.cache.DistributedCache) Preconditions(org.apache.flink.util.Preconditions) StringUtils(org.apache.flink.util.StringUtils) ResourceProfile(org.apache.flink.runtime.clusterframework.types.ResourceProfile) JobExecutionResult(org.apache.flink.api.common.JobExecutionResult) List(java.util.List) ExecutionConfig(org.apache.flink.api.common.ExecutionConfig) Experimental(org.apache.flink.annotation.Experimental) Optional(java.util.Optional) ParallelSourceFunction(org.apache.flink.streaming.api.functions.source.ParallelSourceFunction) Utils(org.apache.flink.api.java.Utils) FileReadFunction(org.apache.flink.streaming.api.functions.source.FileReadFunction) KeyGroupRangeAssignment(org.apache.flink.runtime.state.KeyGroupRangeAssignment) FilePathFilter(org.apache.flink.api.common.io.FilePathFilter) PublicEvolving(org.apache.flink.annotation.PublicEvolving) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) RestartStrategies(org.apache.flink.api.common.restartstrategy.RestartStrategies) DataStreamSource(org.apache.flink.streaming.api.datastream.DataStreamSource) PipelineExecutor(org.apache.flink.core.execution.PipelineExecutor) DeploymentOptions(org.apache.flink.configuration.DeploymentOptions) ContinuousFileReaderOperatorFactory(org.apache.flink.streaming.api.functions.source.ContinuousFileReaderOperatorFactory) Iterator(java.util.Iterator) StateChangelogOptions(org.apache.flink.configuration.StateChangelogOptions) Configuration(org.apache.flink.configuration.Configuration) InputFormatSourceFunction(org.apache.flink.streaming.api.functions.source.InputFormatSourceFunction) DataStream(org.apache.flink.streaming.api.datastream.DataStream) Internal(org.apache.flink.annotation.Internal) FromSplittableIteratorFunction(org.apache.flink.streaming.api.functions.source.FromSplittableIteratorFunction) ClosureCleaner(org.apache.flink.api.java.ClosureCleaner) RuntimeExecutionMode(org.apache.flink.api.common.RuntimeExecutionMode) StreamSource(org.apache.flink.streaming.api.operators.StreamSource) DetachedJobExecutionResult(org.apache.flink.core.execution.DetachedJobExecutionResult) JobExecutionResult(org.apache.flink.api.common.JobExecutionResult) JobClient(org.apache.flink.core.execution.JobClient) DetachedJobExecutionResult(org.apache.flink.core.execution.DetachedJobExecutionResult) Internal(org.apache.flink.annotation.Internal)

Example 3 with DetachedJobExecutionResult

use of org.apache.flink.core.execution.DetachedJobExecutionResult in project flink by apache.

the class StreamContextEnvironment method getJobExecutionResult.

private JobExecutionResult getJobExecutionResult(final JobClient jobClient) throws Exception {
    checkNotNull(jobClient);
    JobExecutionResult jobExecutionResult;
    if (configuration.getBoolean(DeploymentOptions.ATTACHED)) {
        CompletableFuture<JobExecutionResult> jobExecutionResultFuture = jobClient.getJobExecutionResult();
        if (configuration.getBoolean(DeploymentOptions.SHUTDOWN_IF_ATTACHED)) {
            Thread shutdownHook = ShutdownHookUtil.addShutdownHook(() -> {
                // wait a smidgen to allow the async request to go through
                // before
                // the jvm exits
                jobClient.cancel().get(1, TimeUnit.SECONDS);
            }, StreamContextEnvironment.class.getSimpleName(), LOG);
            jobExecutionResultFuture.whenComplete((ignored, throwable) -> ShutdownHookUtil.removeShutdownHook(shutdownHook, StreamContextEnvironment.class.getSimpleName(), LOG));
        }
        jobExecutionResult = jobExecutionResultFuture.get();
        if (!suppressSysout) {
            System.out.println(jobExecutionResult);
        }
    } else {
        jobExecutionResult = new DetachedJobExecutionResult(jobClient.getJobID());
    }
    return jobExecutionResult;
}
Also used : DetachedJobExecutionResult(org.apache.flink.core.execution.DetachedJobExecutionResult) JobExecutionResult(org.apache.flink.api.common.JobExecutionResult) DetachedJobExecutionResult(org.apache.flink.core.execution.DetachedJobExecutionResult)

Aggregations

JobExecutionResult (org.apache.flink.api.common.JobExecutionResult)3 DetachedJobExecutionResult (org.apache.flink.core.execution.DetachedJobExecutionResult)3 Serializer (com.esotericsoftware.kryo.Serializer)1 IOException (java.io.IOException)1 Serializable (java.io.Serializable)1 URI (java.net.URI)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Collection (java.util.Collection)1 HashMap (java.util.HashMap)1 Iterator (java.util.Iterator)1 List (java.util.List)1 Map (java.util.Map)1 Optional (java.util.Optional)1 CompletableFuture (java.util.concurrent.CompletableFuture)1 ExecutionException (java.util.concurrent.ExecutionException)1 Nullable (javax.annotation.Nullable)1 Experimental (org.apache.flink.annotation.Experimental)1 Internal (org.apache.flink.annotation.Internal)1 Public (org.apache.flink.annotation.Public)1