Examples with JobClient org.apache.flink.core.execution.JobClient used on opensource projects

Example 66 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class SinkTestSuiteBase method testBasicSink.

// ----------------------------- Basic test cases ---------------------------------
/**
 * Test DataStream connector sink.
 *
 * <p>The following tests will create a sink in the external system, generate a collection of
 * test data and write them to this sink by the Flink Job.
 *
 * <p>In order to pass these tests, the number of records produced by Flink need to be equals to
 * the generated test data. And the records in the sink will be compared to the test data by the
 * different semantics. There's no requirement for records order.
 */
@TestTemplate
@DisplayName("Test data stream sink")
public void testBasicSink(TestEnvironment testEnv, DataStreamSinkExternalContext<T> externalContext, CheckpointingMode semantic) throws Exception {
    TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
    final List<T> testRecords = generateTestData(sinkSettings, externalContext);
    // Build and execute Flink job
    StreamExecutionEnvironment execEnv = testEnv.createExecutionEnvironment(TestEnvironmentSettings.builder().setConnectorJarPaths(externalContext.getConnectorJarPaths()).build());
    execEnv.enableCheckpointing(50);
    DataStream<T> dataStream = execEnv.fromCollection(testRecords).name("sourceInSinkTest").setParallelism(1).returns(externalContext.getProducedType());
    tryCreateSink(dataStream, externalContext, sinkSettings).setParallelism(1).name("sinkInSinkTest");
    final JobClient jobClient = execEnv.executeAsync("DataStream Sink Test");
    waitForJobStatus(jobClient, Collections.singletonList(JobStatus.FINISHED), Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
    // Check test result
    checkResultWithSemantic(externalContext.createSinkDataReader(sinkSettings), testRecords, semantic);
}

Example 67 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class SinkTestSuiteBase method testMetrics.

/**
 * Test connector sink metrics.
 *
 * <p>This test will create a sink in the external system, generate test data and write them to
 * the sink via a Flink job. Then read and compare the metrics.
 *
 * <p>Now test: numRecordsOut
 */
@TestTemplate
@DisplayName("Test sink metrics")
public void testMetrics(TestEnvironment testEnv, DataStreamSinkExternalContext<T> externalContext, CheckpointingMode semantic) throws Exception {
    TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
    int parallelism = 1;
    final List<T> testRecords = generateTestData(sinkSettings, externalContext);
    // make sure use different names when executes multi times
    String sinkName = "metricTestSink" + testRecords.hashCode();
    final StreamExecutionEnvironment env = testEnv.createExecutionEnvironment(TestEnvironmentSettings.builder().setConnectorJarPaths(externalContext.getConnectorJarPaths()).build());
    env.enableCheckpointing(50);
    DataStreamSource<T> source = env.fromSource(new FromElementsSource<>(Boundedness.CONTINUOUS_UNBOUNDED, testRecords, testRecords.size()), WatermarkStrategy.noWatermarks(), "metricTestSource").setParallelism(1);
    DataStream<T> dataStream = source.returns(externalContext.getProducedType());
    tryCreateSink(dataStream, externalContext, sinkSettings).name(sinkName).setParallelism(parallelism);
    final JobClient jobClient = env.executeAsync("Metrics Test");
    final MetricQuerier queryRestClient = new MetricQuerier(new Configuration());
    final ExecutorService executorService = Executors.newCachedThreadPool();
    try {
        waitForAllTaskRunning(() -> getJobDetails(new RestClient(new Configuration(), executorService), testEnv.getRestEndpoint(), jobClient.getJobID()), Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
        waitUntilCondition(() -> {
            // test metrics
            try {
                return compareSinkMetrics(queryRestClient, testEnv, externalContext, jobClient.getJobID(), sinkName, testRecords.size());
            } catch (Exception e) {
                // skip failed assert try
                return false;
            }
        }, Deadline.fromNow(DEFAULT_COLLECT_DATA_TIMEOUT));
    } finally {
        // Clean up
        executorService.shutdown();
        killJob(jobClient);
    }
}

Example 68 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class SinkTestSuiteBase method restartFromSavepoint.

private void restartFromSavepoint(TestEnvironment testEnv, DataStreamSinkExternalContext<T> externalContext, CheckpointingMode semantic, final int beforeParallelism, final int afterParallelism) throws Exception {
    // Step 1: Preparation
    TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
    final StreamExecutionEnvironment execEnv = testEnv.createExecutionEnvironment(TestEnvironmentSettings.builder().setConnectorJarPaths(externalContext.getConnectorJarPaths()).build());
    execEnv.setRestartStrategy(RestartStrategies.noRestart());
    // Step 2: Generate test data
    final List<T> testRecords = generateTestData(sinkSettings, externalContext);
    // Step 3: Build and execute Flink job
    int numBeforeSuccess = testRecords.size() / 2;
    DataStreamSource<T> source = execEnv.fromSource(new FromElementsSource<>(Boundedness.CONTINUOUS_UNBOUNDED, testRecords, numBeforeSuccess), WatermarkStrategy.noWatermarks(), "beforeRestartSource").setParallelism(1);
    DataStream<T> dataStream = source.returns(externalContext.getProducedType());
    tryCreateSink(dataStream, externalContext, sinkSettings).name("Sink restart test").setParallelism(beforeParallelism);
    /**
     * The job should stop after consume a specified number of records. In order to know when
     * the specified number of records have been consumed, a collect sink is need to be watched.
     */
    CollectResultIterator<T> iterator = addCollectSink(source);
    final JobClient jobClient = execEnv.executeAsync("Restart Test");
    iterator.setJobClient(jobClient);
    // Step 4: Wait for the expected result and stop Flink job with a savepoint
    final ExecutorService executorService = Executors.newCachedThreadPool();
    String savepointPath;
    try {
        waitForAllTaskRunning(() -> getJobDetails(new RestClient(new Configuration(), executorService), testEnv.getRestEndpoint(), jobClient.getJobID()), Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
        waitExpectedSizeData(iterator, numBeforeSuccess);
        savepointPath = jobClient.stopWithSavepoint(true, testEnv.getCheckpointUri(), SavepointFormatType.CANONICAL).get(30, TimeUnit.SECONDS);
        waitForJobStatus(jobClient, Collections.singletonList(JobStatus.FINISHED), Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
    } catch (Exception e) {
        executorService.shutdown();
        killJob(jobClient);
        throw e;
    }
    List<T> target = testRecords.subList(0, numBeforeSuccess);
    checkResultWithSemantic(externalContext.createSinkDataReader(sinkSettings), target, semantic);
    // Step 4: restart the Flink job with the savepoint
    final StreamExecutionEnvironment restartEnv = testEnv.createExecutionEnvironment(TestEnvironmentSettings.builder().setConnectorJarPaths(externalContext.getConnectorJarPaths()).setSavepointRestorePath(savepointPath).build());
    restartEnv.enableCheckpointing(50);
    DataStreamSource<T> restartSource = restartEnv.fromSource(new FromElementsSource<>(Boundedness.CONTINUOUS_UNBOUNDED, testRecords, testRecords.size()), WatermarkStrategy.noWatermarks(), "restartSource").setParallelism(1);
    DataStream<T> sinkStream = restartSource.returns(externalContext.getProducedType());
    tryCreateSink(sinkStream, externalContext, sinkSettings).setParallelism(afterParallelism);
    addCollectSink(restartSource);
    final JobClient restartJobClient = restartEnv.executeAsync("Restart Test");
    try {
        // Check the result
        checkResultWithSemantic(externalContext.createSinkDataReader(sinkSettings), testRecords, semantic);
    } finally {
        executorService.shutdown();
        killJob(restartJobClient);
        iterator.close();
    }
}

Example 69 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class LocalRecoveryITCase method submitJob.

private JobClient submitJob(int parallelism, StandaloneSessionClusterEntrypoint clusterEntrypoint) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", clusterEntrypoint.getRestPort(), new Configuration());
    env.setParallelism(parallelism);
    env.enableCheckpointing(100, CheckpointingMode.EXACTLY_ONCE);
    env.addSource(new LocalRecoverySource()).keyBy(x -> x).addSink(new DiscardingSink<>());
    final JobClient jobClient = env.executeAsync();
    return jobClient;
}

Example 70 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class LocalRecoveryITCase method testRecoverLocallyFromProcessCrashWithWorkingDirectory.

@Test
public void testRecoverLocallyFromProcessCrashWithWorkingDirectory() throws Exception {
    final Configuration configuration = new Configuration();
    configuration.set(JobManagerOptions.ADDRESS, "localhost");
    configuration.set(JobManagerOptions.PORT, 0);
    configuration.set(RestOptions.BIND_PORT, "0");
    configuration.set(HeartbeatManagerOptions.HEARTBEAT_TIMEOUT, 10000L);
    configuration.set(HeartbeatManagerOptions.HEARTBEAT_INTERVAL, 1000L);
    configuration.set(HeartbeatManagerOptions.HEARTBEAT_RPC_FAILURE_THRESHOLD, 1);
    configuration.set(ClusterOptions.PROCESS_WORKING_DIR_BASE, tmpDirectory.getAbsolutePath());
    configuration.set(CheckpointingOptions.LOCAL_RECOVERY, true);
    configuration.set(TaskManagerOptions.SLOT_TIMEOUT, Duration.ofSeconds(30L));
    final int parallelism = 3;
    boolean success = false;
    Collection<TaskManagerProcess> taskManagerProcesses = Collections.emptyList();
    try (final StandaloneSessionClusterEntrypoint clusterEntrypoint = new StandaloneSessionClusterEntrypoint(configuration)) {
        clusterEntrypoint.startCluster();
        final Configuration configurationTemplate = new Configuration(configuration);
        configurationTemplate.set(JobManagerOptions.PORT, clusterEntrypoint.getRpcPort());
        taskManagerProcesses = startTaskManagerProcesses(parallelism, configurationTemplate);
        final JobClient jobClient = submitJob(parallelism, clusterEntrypoint);
        final long waitingTimeInSeconds = 45L;
        waitUntilCheckpointCompleted(configuration, clusterEntrypoint.getRestPort(), jobClient.getJobID(), Deadline.fromNow(Duration.ofSeconds(waitingTimeInSeconds)));
        restartTaskManagerProcesses(taskManagerProcesses, parallelism - 1);
        jobClient.getJobExecutionResult().get(waitingTimeInSeconds, TimeUnit.SECONDS);
        success = true;
    } finally {
        if (!success) {
            for (TaskManagerProcess taskManagerProcess : taskManagerProcesses) {
                printLogOutput(taskManagerProcess);
            }
        }
        for (TaskManagerProcess taskManagerProcess : taskManagerProcesses) {
            taskManagerProcess.terminate();
        }
    }
}