Examples with JobListener org.apache.flink.core.execution.JobListener used on opensource projects

Example 6 with JobListener

use of org.apache.flink.core.execution.JobListener in project flink by apache.

the class JobListenerITCase method testExecuteCallsJobListenerOnMainThreadOnStreamEnvironment.

@Test
public void testExecuteCallsJobListenerOnMainThreadOnStreamEnvironment() throws Exception {
    AtomicReference<Thread> threadReference = new AtomicReference<>();
    StreamExecutionEnvironment env = new StreamExecutionEnvironment(getClientConfiguration());
    env.registerJobListener(new JobListener() {

        @Override
        public void onJobSubmitted(JobClient jobClient, Throwable t) {
            threadReference.set(Thread.currentThread());
        }

        @Override
        public void onJobExecuted(JobExecutionResult jobExecutionResult, Throwable throwable) {
        }
    });
    env.fromElements(1, 2, 3, 4, 5).addSink(new DiscardingSink<>());
    env.execute();
    assertThat(Thread.currentThread(), is(threadReference.get()));
}

Example 7 with JobListener

use of org.apache.flink.core.execution.JobListener in project flink by apache.

the class JobListenerITCase method testExecuteAsyncCallsJobListenerOnBatchEnvironment.

@Test
public void testExecuteAsyncCallsJobListenerOnBatchEnvironment() throws Exception {
    AtomicReference<JobID> jobIdReference = new AtomicReference<>();
    OneShotLatch submissionLatch = new OneShotLatch();
    ExecutionEnvironment env = new ExecutionEnvironment(getClientConfiguration());
    env.registerJobListener(new JobListener() {

        @Override
        public void onJobSubmitted(JobClient jobClient, Throwable t) {
            jobIdReference.set(jobClient.getJobID());
            submissionLatch.trigger();
        }

        @Override
        public void onJobExecuted(JobExecutionResult jobExecutionResult, Throwable throwable) {
        }
    });
    env.fromElements(1, 2, 3, 4, 5).output(new DiscardingOutputFormat<>());
    JobClient jobClient = env.executeAsync();
    submissionLatch.await(2000L, TimeUnit.MILLISECONDS);
    // when executing asynchronously we don't get an "executed" callback
    assertThat(jobClient.getJobID(), is(jobIdReference.get()));
}

Example 8 with JobListener

use of org.apache.flink.core.execution.JobListener in project flink by apache.

the class StreamExecutionEnvironment method execute.

/**
 * Triggers the program execution. The environment will execute all parts of the program that
 * have resulted in a "sink" operation. Sink operations are for example printing results or
 * forwarding them to a message queue.
 *
 * @param streamGraph the stream graph representing the transformations
 * @return The result of the job execution, containing elapsed time and accumulators.
 * @throws Exception which occurs during job execution.
 */
@Internal
public JobExecutionResult execute(StreamGraph streamGraph) throws Exception {
    final JobClient jobClient = executeAsync(streamGraph);
    try {
        final JobExecutionResult jobExecutionResult;
        if (configuration.getBoolean(DeploymentOptions.ATTACHED)) {
            jobExecutionResult = jobClient.getJobExecutionResult().get();
        } else {
            jobExecutionResult = new DetachedJobExecutionResult(jobClient.getJobID());
        }
        jobListeners.forEach(jobListener -> jobListener.onJobExecuted(jobExecutionResult, null));
        return jobExecutionResult;
    } catch (Throwable t) {
        // get() on the JobExecutionResult Future will throw an ExecutionException. This
        // behaviour was largely not there in Flink versions before the PipelineExecutor
        // refactoring so we should strip that exception.
        Throwable strippedException = ExceptionUtils.stripExecutionException(t);
        jobListeners.forEach(jobListener -> {
            jobListener.onJobExecuted(null, strippedException);
        });
        ExceptionUtils.rethrowException(strippedException);
        // never reached, only make javac happy
        return null;
    }
}

Example 9 with JobListener

use of org.apache.flink.core.execution.JobListener in project flink by apache.

the class StreamExecutionEnvironment method registerCustomListeners.

private void registerCustomListeners(final ClassLoader classLoader, final List<String> listeners) {
    for (String listener : listeners) {
        try {
            final JobListener jobListener = InstantiationUtil.instantiate(listener, JobListener.class, classLoader);
            jobListeners.add(jobListener);
        } catch (FlinkException e) {
            throw new WrappingRuntimeException("Could not load JobListener : " + listener, e);
        }
    }
}

Example 10 with JobListener

use of org.apache.flink.core.execution.JobListener in project flink by apache.

the class StreamExecutionEnvironment method executeAsync.

/**
 * Triggers the program execution asynchronously. The environment will execute all parts of the
 * program that have resulted in a "sink" operation. Sink operations are for example printing
 * results or forwarding them to a message queue.
 *
 * @param streamGraph the stream graph representing the transformations
 * @return A {@link JobClient} that can be used to communicate with the submitted job, completed
 *     on submission succeeded.
 * @throws Exception which occurs during job execution.
 */
@Internal
public JobClient executeAsync(StreamGraph streamGraph) throws Exception {
    checkNotNull(streamGraph, "StreamGraph cannot be null.");
    checkNotNull(configuration.get(DeploymentOptions.TARGET), "No execution.target specified in your configuration file.");
    final PipelineExecutorFactory executorFactory = executorServiceLoader.getExecutorFactory(configuration);
    checkNotNull(executorFactory, "Cannot find compatible factory for specified execution.target (=%s)", configuration.get(DeploymentOptions.TARGET));
    CompletableFuture<JobClient> jobClientFuture = executorFactory.getExecutor(configuration).execute(streamGraph, configuration, userClassloader);
    try {
        JobClient jobClient = jobClientFuture.get();
        jobListeners.forEach(jobListener -> jobListener.onJobSubmitted(jobClient, null));
        return jobClient;
    } catch (ExecutionException executionException) {
        final Throwable strippedException = ExceptionUtils.stripExecutionException(executionException);
        jobListeners.forEach(jobListener -> jobListener.onJobSubmitted(null, strippedException));
        throw new FlinkException(String.format("Failed to execute job '%s'.", streamGraph.getJobName()), strippedException);
    }
}