Search in sources :

Example 1 with PipelineExecutor

use of org.apache.flink.core.execution.PipelineExecutor in project flink by apache.

the class StreamExecutionEnvironment method execute.

/**
 * Triggers the program execution. The environment will execute all parts of the program that
 * have resulted in a "sink" operation. Sink operations are for example printing results or
 * forwarding them to a message queue.
 *
 * @param streamGraph the stream graph representing the transformations
 * @return The result of the job execution, containing elapsed time and accumulators.
 * @throws Exception which occurs during job execution.
 */
@Internal
public JobExecutionResult execute(StreamGraph streamGraph) throws Exception {
    final JobClient jobClient = executeAsync(streamGraph);
    try {
        final JobExecutionResult jobExecutionResult;
        if (configuration.getBoolean(DeploymentOptions.ATTACHED)) {
            jobExecutionResult = jobClient.getJobExecutionResult().get();
        } else {
            jobExecutionResult = new DetachedJobExecutionResult(jobClient.getJobID());
        }
        jobListeners.forEach(jobListener -> jobListener.onJobExecuted(jobExecutionResult, null));
        return jobExecutionResult;
    } catch (Throwable t) {
        // get() on the JobExecutionResult Future will throw an ExecutionException. This
        // behaviour was largely not there in Flink versions before the PipelineExecutor
        // refactoring so we should strip that exception.
        Throwable strippedException = ExceptionUtils.stripExecutionException(t);
        jobListeners.forEach(jobListener -> {
            jobListener.onJobExecuted(null, strippedException);
        });
        ExceptionUtils.rethrowException(strippedException);
        // never reached, only make javac happy
        return null;
    }
}
Also used : Arrays(java.util.Arrays) SocketTextStreamFunction(org.apache.flink.streaming.api.functions.source.SocketTextStreamFunction) Tuple3(org.apache.flink.api.java.tuple.Tuple3) Tuple2(org.apache.flink.api.java.tuple.Tuple2) FromIteratorFunction(org.apache.flink.streaming.api.functions.source.FromIteratorFunction) StreamGraphGenerator(org.apache.flink.streaming.api.graph.StreamGraphGenerator) MemorySize(org.apache.flink.configuration.MemorySize) StateBackend(org.apache.flink.runtime.state.StateBackend) WrappingRuntimeException(org.apache.flink.util.WrappingRuntimeException) SplittableIterator(org.apache.flink.util.SplittableIterator) InvalidProgramException(org.apache.flink.api.common.InvalidProgramException) StateBackendLoader(org.apache.flink.runtime.state.StateBackendLoader) Path(org.apache.flink.core.fs.Path) ResourceSpec(org.apache.flink.api.common.operators.ResourceSpec) Map(java.util.Map) StatefulSequenceSource(org.apache.flink.streaming.api.functions.source.StatefulSequenceSource) StreamGraph(org.apache.flink.streaming.api.graph.StreamGraph) FileMonitoringFunction(org.apache.flink.streaming.api.functions.source.FileMonitoringFunction) DefaultExecutorServiceLoader(org.apache.flink.core.execution.DefaultExecutorServiceLoader) ContinuousFileMonitoringFunction(org.apache.flink.streaming.api.functions.source.ContinuousFileMonitoringFunction) PipelineExecutorFactory(org.apache.flink.core.execution.PipelineExecutorFactory) FromElementsFunction(org.apache.flink.streaming.api.functions.source.FromElementsFunction) TernaryBoolean(org.apache.flink.util.TernaryBoolean) Serializable(java.io.Serializable) TypeExtractor(org.apache.flink.api.java.typeutils.TypeExtractor) ResultTypeQueryable(org.apache.flink.api.java.typeutils.ResultTypeQueryable) Source(org.apache.flink.api.connector.source.Source) Time(org.apache.flink.api.common.time.Time) Boundedness(org.apache.flink.api.connector.source.Boundedness) FlinkException(org.apache.flink.util.FlinkException) NumberSequenceSource(org.apache.flink.api.connector.source.lib.NumberSequenceSource) PipelineExecutorServiceLoader(org.apache.flink.core.execution.PipelineExecutorServiceLoader) SlotSharingGroupUtils(org.apache.flink.api.common.operators.util.SlotSharingGroupUtils) UnmodifiableConfiguration(org.apache.flink.configuration.UnmodifiableConfiguration) ArrayList(java.util.ArrayList) SourceFunction(org.apache.flink.streaming.api.functions.source.SourceFunction) ReadableConfig(org.apache.flink.configuration.ReadableConfig) JobListener(org.apache.flink.core.execution.JobListener) InvalidTypesException(org.apache.flink.api.common.functions.InvalidTypesException) RestOptions(org.apache.flink.configuration.RestOptions) ConfigOption(org.apache.flink.configuration.ConfigOption) Nullable(javax.annotation.Nullable) DetachedJobExecutionResult(org.apache.flink.core.execution.DetachedJobExecutionResult) TimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic) Serializer(com.esotericsoftware.kryo.Serializer) SingleOutputStreamOperator(org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator) MissingTypeInfo(org.apache.flink.api.java.typeutils.MissingTypeInfo) IOException(java.io.IOException) JobClient(org.apache.flink.core.execution.JobClient) DynamicCodeLoadingException(org.apache.flink.util.DynamicCodeLoadingException) ExecutionException(java.util.concurrent.ExecutionException) TextInputFormat(org.apache.flink.api.java.io.TextInputFormat) CoreOptions(org.apache.flink.configuration.CoreOptions) Transformation(org.apache.flink.api.dag.Transformation) PojoTypeInfo(org.apache.flink.api.java.typeutils.PojoTypeInfo) FileProcessingMode(org.apache.flink.streaming.api.functions.source.FileProcessingMode) PipelineOptions(org.apache.flink.configuration.PipelineOptions) CheckpointingMode(org.apache.flink.streaming.api.CheckpointingMode) ExceptionUtils(org.apache.flink.util.ExceptionUtils) SlotSharingGroup(org.apache.flink.api.common.operators.SlotSharingGroup) FileInputFormat(org.apache.flink.api.common.io.FileInputFormat) BasicTypeInfo(org.apache.flink.api.common.typeinfo.BasicTypeInfo) InstantiationUtil(org.apache.flink.util.InstantiationUtil) InputFormat(org.apache.flink.api.common.io.InputFormat) Public(org.apache.flink.annotation.Public) URI(java.net.URI) Preconditions.checkNotNull(org.apache.flink.util.Preconditions.checkNotNull) TypeInformation(org.apache.flink.api.common.typeinfo.TypeInformation) ExecutionOptions(org.apache.flink.configuration.ExecutionOptions) TimestampedFileInputSplit(org.apache.flink.streaming.api.functions.source.TimestampedFileInputSplit) Collection(java.util.Collection) WatermarkStrategy(org.apache.flink.api.common.eventtime.WatermarkStrategy) DistributedCache(org.apache.flink.api.common.cache.DistributedCache) Preconditions(org.apache.flink.util.Preconditions) StringUtils(org.apache.flink.util.StringUtils) ResourceProfile(org.apache.flink.runtime.clusterframework.types.ResourceProfile) JobExecutionResult(org.apache.flink.api.common.JobExecutionResult) List(java.util.List) ExecutionConfig(org.apache.flink.api.common.ExecutionConfig) Experimental(org.apache.flink.annotation.Experimental) Optional(java.util.Optional) ParallelSourceFunction(org.apache.flink.streaming.api.functions.source.ParallelSourceFunction) Utils(org.apache.flink.api.java.Utils) FileReadFunction(org.apache.flink.streaming.api.functions.source.FileReadFunction) KeyGroupRangeAssignment(org.apache.flink.runtime.state.KeyGroupRangeAssignment) FilePathFilter(org.apache.flink.api.common.io.FilePathFilter) PublicEvolving(org.apache.flink.annotation.PublicEvolving) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) RestartStrategies(org.apache.flink.api.common.restartstrategy.RestartStrategies) DataStreamSource(org.apache.flink.streaming.api.datastream.DataStreamSource) PipelineExecutor(org.apache.flink.core.execution.PipelineExecutor) DeploymentOptions(org.apache.flink.configuration.DeploymentOptions) ContinuousFileReaderOperatorFactory(org.apache.flink.streaming.api.functions.source.ContinuousFileReaderOperatorFactory) Iterator(java.util.Iterator) StateChangelogOptions(org.apache.flink.configuration.StateChangelogOptions) Configuration(org.apache.flink.configuration.Configuration) InputFormatSourceFunction(org.apache.flink.streaming.api.functions.source.InputFormatSourceFunction) DataStream(org.apache.flink.streaming.api.datastream.DataStream) Internal(org.apache.flink.annotation.Internal) FromSplittableIteratorFunction(org.apache.flink.streaming.api.functions.source.FromSplittableIteratorFunction) ClosureCleaner(org.apache.flink.api.java.ClosureCleaner) RuntimeExecutionMode(org.apache.flink.api.common.RuntimeExecutionMode) StreamSource(org.apache.flink.streaming.api.operators.StreamSource) DetachedJobExecutionResult(org.apache.flink.core.execution.DetachedJobExecutionResult) JobExecutionResult(org.apache.flink.api.common.JobExecutionResult) JobClient(org.apache.flink.core.execution.JobClient) DetachedJobExecutionResult(org.apache.flink.core.execution.DetachedJobExecutionResult) Internal(org.apache.flink.annotation.Internal)

Aggregations

Serializer (com.esotericsoftware.kryo.Serializer)1 IOException (java.io.IOException)1 Serializable (java.io.Serializable)1 URI (java.net.URI)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Collection (java.util.Collection)1 HashMap (java.util.HashMap)1 Iterator (java.util.Iterator)1 List (java.util.List)1 Map (java.util.Map)1 Optional (java.util.Optional)1 CompletableFuture (java.util.concurrent.CompletableFuture)1 ExecutionException (java.util.concurrent.ExecutionException)1 Nullable (javax.annotation.Nullable)1 Experimental (org.apache.flink.annotation.Experimental)1 Internal (org.apache.flink.annotation.Internal)1 Public (org.apache.flink.annotation.Public)1 PublicEvolving (org.apache.flink.annotation.PublicEvolving)1 ExecutionConfig (org.apache.flink.api.common.ExecutionConfig)1