Examples with JobClient org.apache.flink.core.execution.JobClient used on opensource projects

Example 21 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class SinkMetricsITCase method testMetrics.

@Test
public void testMetrics() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    int numSplits = Math.max(1, env.getParallelism() - 2);
    int numRecordsPerSplit = 10;
    // make sure all parallel instances have processed the same amount of records before
    // validating metrics
    SharedReference<CyclicBarrier> beforeBarrier = sharedObjects.add(new CyclicBarrier(numSplits + 1));
    SharedReference<CyclicBarrier> afterBarrier = sharedObjects.add(new CyclicBarrier(numSplits + 1));
    int stopAtRecord1 = 4;
    int stopAtRecord2 = numRecordsPerSplit - 1;
    env.fromSequence(0, numSplits - 1).<Long>flatMap((split, collector) -> LongStream.range(0, numRecordsPerSplit).forEach(collector::collect)).returns(BasicTypeInfo.LONG_TYPE_INFO).map(i -> {
        if (i % numRecordsPerSplit == stopAtRecord1 || i % numRecordsPerSplit == stopAtRecord2) {
            beforeBarrier.get().await();
            afterBarrier.get().await();
        }
        return i;
    }).sinkTo(TestSink.newBuilder().setWriter(new MetricWriter()).build()).name("MetricTestSink");
    JobClient jobClient = env.executeAsync();
    final JobID jobId = jobClient.getJobID();
    beforeBarrier.get().await();
    assertSinkMetrics(jobId, stopAtRecord1, env.getParallelism(), numSplits);
    afterBarrier.get().await();
    beforeBarrier.get().await();
    assertSinkMetrics(jobId, stopAtRecord2, env.getParallelism(), numSplits);
    afterBarrier.get().await();
    jobClient.getJobExecutionResult().get();
}

Example 22 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class DataStream method executeAndCollectWithClient.

ClientAndIterator<T> executeAndCollectWithClient(String jobExecutionName) throws Exception {
    TypeSerializer<T> serializer = getType().createSerializer(getExecutionEnvironment().getConfig());
    String accumulatorName = "dataStreamCollect_" + UUID.randomUUID().toString();
    StreamExecutionEnvironment env = getExecutionEnvironment();
    CollectSinkOperatorFactory<T> factory = new CollectSinkOperatorFactory<>(serializer, accumulatorName);
    CollectSinkOperator<T> operator = (CollectSinkOperator<T>) factory.getOperator();
    CollectResultIterator<T> iterator = new CollectResultIterator<>(operator.getOperatorIdFuture(), serializer, accumulatorName, env.getCheckpointConfig());
    CollectStreamSink<T> sink = new CollectStreamSink<>(this, factory);
    sink.name("Data stream collect sink");
    env.addOperator(sink.getTransformation());
    final JobClient jobClient = env.executeAsync(jobExecutionName);
    iterator.setJobClient(jobClient);
    return new ClientAndIterator<>(jobClient, iterator);
}

Example 23 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class SourceTestSuiteBase method restartFromSavepoint.

private void restartFromSavepoint(TestEnvironment testEnv, DataStreamSourceExternalContext<T> externalContext, CheckpointingMode semantic, final int splitNumber, final int beforeParallelism, final int afterParallelism) throws Exception {
    // Step 1: Preparation
    TestingSourceSettings sourceSettings = TestingSourceSettings.builder().setBoundedness(Boundedness.CONTINUOUS_UNBOUNDED).setCheckpointingMode(semantic).build();
    TestEnvironmentSettings envOptions = TestEnvironmentSettings.builder().setConnectorJarPaths(externalContext.getConnectorJarPaths()).build();
    // Step 2: Generate test data
    final List<ExternalSystemSplitDataWriter<T>> writers = new ArrayList<>();
    final List<List<T>> testRecordCollections = new ArrayList<>();
    for (int i = 0; i < splitNumber; i++) {
        writers.add(externalContext.createSourceSplitDataWriter(sourceSettings));
        testRecordCollections.add(generateTestDataForWriter(externalContext, sourceSettings, i, writers.get(i)));
    }
    // Step 3: Build and execute Flink job
    final StreamExecutionEnvironment execEnv = testEnv.createExecutionEnvironment(envOptions);
    execEnv.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
    execEnv.enableCheckpointing(50);
    execEnv.setRestartStrategy(RestartStrategies.noRestart());
    DataStreamSource<T> source = execEnv.fromSource(tryCreateSource(externalContext, sourceSettings), WatermarkStrategy.noWatermarks(), "Tested Source").setParallelism(beforeParallelism);
    CollectIteratorBuilder<T> iteratorBuilder = addCollectSink(source);
    final JobClient jobClient = execEnv.executeAsync("Restart Test");
    // Step 4: Check the result and stop Flink job with a savepoint
    CollectResultIterator<T> iterator = null;
    try {
        iterator = iteratorBuilder.build(jobClient);
        checkResultWithSemantic(iterator, testRecordCollections, semantic, getTestDataSize(testRecordCollections));
    } catch (Exception e) {
        killJob(jobClient);
        throw e;
    }
    String savepointPath = jobClient.stopWithSavepoint(true, testEnv.getCheckpointUri(), SavepointFormatType.CANONICAL).get(30, TimeUnit.SECONDS);
    waitForJobStatus(jobClient, Collections.singletonList(JobStatus.FINISHED), Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
    // Step 5: Generate new test data
    final List<List<T>> newTestRecordCollections = new ArrayList<>();
    for (int i = 0; i < splitNumber; i++) {
        newTestRecordCollections.add(generateTestDataForWriter(externalContext, sourceSettings, i, writers.get(i)));
    }
    // Step 6: restart the Flink job with the savepoint
    TestEnvironmentSettings restartEnvOptions = TestEnvironmentSettings.builder().setConnectorJarPaths(externalContext.getConnectorJarPaths()).setSavepointRestorePath(savepointPath).build();
    final StreamExecutionEnvironment restartEnv = testEnv.createExecutionEnvironment(restartEnvOptions);
    restartEnv.enableCheckpointing(500);
    restartEnv.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
    DataStreamSource<T> restartSource = restartEnv.fromSource(tryCreateSource(externalContext, sourceSettings), WatermarkStrategy.noWatermarks(), "Tested Source").setParallelism(afterParallelism);
    addCollectSink(restartSource);
    final JobClient restartJobClient = restartEnv.executeAsync("Restart Test");
    waitForJobStatus(restartJobClient, Collections.singletonList(JobStatus.RUNNING), Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
    try {
        iterator.setJobClient(restartJobClient);
        /*
             * Use the same iterator as the previous run, because the CollectStreamSink will snapshot
             * its state and recover from it.
             *
             * The fetcher in CollectResultIterator is responsible for comminicating with
             * the CollectSinkFunction, and deal the result with CheckpointedCollectResultBuffer
             * in EXACTLY_ONCE semantic.
             */
        checkResultWithSemantic(iterator, newTestRecordCollections, semantic, getTestDataSize(newTestRecordCollections));
    } finally {
        // Clean up
        killJob(restartJobClient);
        iterator.close();
    }
}

Example 24 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class StreamExecutionEnvironment method execute.

/**
 * Triggers the program execution. The environment will execute all parts of the program that
 * have resulted in a "sink" operation. Sink operations are for example printing results or
 * forwarding them to a message queue.
 *
 * @param streamGraph the stream graph representing the transformations
 * @return The result of the job execution, containing elapsed time and accumulators.
 * @throws Exception which occurs during job execution.
 */
@Internal
public JobExecutionResult execute(StreamGraph streamGraph) throws Exception {
    final JobClient jobClient = executeAsync(streamGraph);
    try {
        final JobExecutionResult jobExecutionResult;
        if (configuration.getBoolean(DeploymentOptions.ATTACHED)) {
            jobExecutionResult = jobClient.getJobExecutionResult().get();
        } else {
            jobExecutionResult = new DetachedJobExecutionResult(jobClient.getJobID());
        }
        jobListeners.forEach(jobListener -> jobListener.onJobExecuted(jobExecutionResult, null));
        return jobExecutionResult;
    } catch (Throwable t) {
        // get() on the JobExecutionResult Future will throw an ExecutionException. This
        // behaviour was largely not there in Flink versions before the PipelineExecutor
        // refactoring so we should strip that exception.
        Throwable strippedException = ExceptionUtils.stripExecutionException(t);
        jobListeners.forEach(jobListener -> {
            jobListener.onJobExecuted(null, strippedException);
        });
        ExceptionUtils.rethrowException(strippedException);
        // never reached, only make javac happy
        return null;
    }
}

Example 25 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class StreamExecutionEnvironment method executeAsync.

/**
 * Triggers the program execution asynchronously. The environment will execute all parts of the
 * program that have resulted in a "sink" operation. Sink operations are for example printing
 * results or forwarding them to a message queue.
 *
 * @param streamGraph the stream graph representing the transformations
 * @return A {@link JobClient} that can be used to communicate with the submitted job, completed
 *     on submission succeeded.
 * @throws Exception which occurs during job execution.
 */
@Internal
public JobClient executeAsync(StreamGraph streamGraph) throws Exception {
    checkNotNull(streamGraph, "StreamGraph cannot be null.");
    checkNotNull(configuration.get(DeploymentOptions.TARGET), "No execution.target specified in your configuration file.");
    final PipelineExecutorFactory executorFactory = executorServiceLoader.getExecutorFactory(configuration);
    checkNotNull(executorFactory, "Cannot find compatible factory for specified execution.target (=%s)", configuration.get(DeploymentOptions.TARGET));
    CompletableFuture<JobClient> jobClientFuture = executorFactory.getExecutor(configuration).execute(streamGraph, configuration, userClassloader);
    try {
        JobClient jobClient = jobClientFuture.get();
        jobListeners.forEach(jobListener -> jobListener.onJobSubmitted(jobClient, null));
        return jobClient;
    } catch (ExecutionException executionException) {
        final Throwable strippedException = ExceptionUtils.stripExecutionException(executionException);
        jobListeners.forEach(jobListener -> jobListener.onJobSubmitted(null, strippedException));
        throw new FlinkException(String.format("Failed to execute job '%s'.", streamGraph.getJobName()), strippedException);
    }
}