Examples with FromElementsSource org.apache.flink.connector.testframe.source.FromElementsSource used on opensource projects

Example 1 with FromElementsSource

use of org.apache.flink.connector.testframe.source.FromElementsSource in project flink by apache.

the class SinkTestSuiteBase method testMetrics.

/**
 * Test connector sink metrics.
 *
 * <p>This test will create a sink in the external system, generate test data and write them to
 * the sink via a Flink job. Then read and compare the metrics.
 *
 * <p>Now test: numRecordsOut
 */
@TestTemplate
@DisplayName("Test sink metrics")
public void testMetrics(TestEnvironment testEnv, DataStreamSinkExternalContext<T> externalContext, CheckpointingMode semantic) throws Exception {
    TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
    int parallelism = 1;
    final List<T> testRecords = generateTestData(sinkSettings, externalContext);
    // make sure use different names when executes multi times
    String sinkName = "metricTestSink" + testRecords.hashCode();
    final StreamExecutionEnvironment env = testEnv.createExecutionEnvironment(TestEnvironmentSettings.builder().setConnectorJarPaths(externalContext.getConnectorJarPaths()).build());
    env.enableCheckpointing(50);
    DataStreamSource<T> source = env.fromSource(new FromElementsSource<>(Boundedness.CONTINUOUS_UNBOUNDED, testRecords, testRecords.size()), WatermarkStrategy.noWatermarks(), "metricTestSource").setParallelism(1);
    DataStream<T> dataStream = source.returns(externalContext.getProducedType());
    tryCreateSink(dataStream, externalContext, sinkSettings).name(sinkName).setParallelism(parallelism);
    final JobClient jobClient = env.executeAsync("Metrics Test");
    final MetricQuerier queryRestClient = new MetricQuerier(new Configuration());
    final ExecutorService executorService = Executors.newCachedThreadPool();
    try {
        waitForAllTaskRunning(() -> getJobDetails(new RestClient(new Configuration(), executorService), testEnv.getRestEndpoint(), jobClient.getJobID()), Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
        waitUntilCondition(() -> {
            // test metrics
            try {
                return compareSinkMetrics(queryRestClient, testEnv, externalContext, jobClient.getJobID(), sinkName, testRecords.size());
            } catch (Exception e) {
                // skip failed assert try
                return false;
            }
        }, Deadline.fromNow(DEFAULT_COLLECT_DATA_TIMEOUT));
    } finally {
        // Clean up
        executorService.shutdown();
        killJob(jobClient);
    }
}

Example 2 with FromElementsSource

use of org.apache.flink.connector.testframe.source.FromElementsSource in project flink by apache.

the class SinkTestSuiteBase method restartFromSavepoint.

private void restartFromSavepoint(TestEnvironment testEnv, DataStreamSinkExternalContext<T> externalContext, CheckpointingMode semantic, final int beforeParallelism, final int afterParallelism) throws Exception {
    // Step 1: Preparation
    TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
    final StreamExecutionEnvironment execEnv = testEnv.createExecutionEnvironment(TestEnvironmentSettings.builder().setConnectorJarPaths(externalContext.getConnectorJarPaths()).build());
    execEnv.setRestartStrategy(RestartStrategies.noRestart());
    // Step 2: Generate test data
    final List<T> testRecords = generateTestData(sinkSettings, externalContext);
    // Step 3: Build and execute Flink job
    int numBeforeSuccess = testRecords.size() / 2;
    DataStreamSource<T> source = execEnv.fromSource(new FromElementsSource<>(Boundedness.CONTINUOUS_UNBOUNDED, testRecords, numBeforeSuccess), WatermarkStrategy.noWatermarks(), "beforeRestartSource").setParallelism(1);
    DataStream<T> dataStream = source.returns(externalContext.getProducedType());
    tryCreateSink(dataStream, externalContext, sinkSettings).name("Sink restart test").setParallelism(beforeParallelism);
    /**
     * The job should stop after consume a specified number of records. In order to know when
     * the specified number of records have been consumed, a collect sink is need to be watched.
     */
    CollectResultIterator<T> iterator = addCollectSink(source);
    final JobClient jobClient = execEnv.executeAsync("Restart Test");
    iterator.setJobClient(jobClient);
    // Step 4: Wait for the expected result and stop Flink job with a savepoint
    final ExecutorService executorService = Executors.newCachedThreadPool();
    String savepointPath;
    try {
        waitForAllTaskRunning(() -> getJobDetails(new RestClient(new Configuration(), executorService), testEnv.getRestEndpoint(), jobClient.getJobID()), Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
        waitExpectedSizeData(iterator, numBeforeSuccess);
        savepointPath = jobClient.stopWithSavepoint(true, testEnv.getCheckpointUri(), SavepointFormatType.CANONICAL).get(30, TimeUnit.SECONDS);
        waitForJobStatus(jobClient, Collections.singletonList(JobStatus.FINISHED), Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
    } catch (Exception e) {
        executorService.shutdown();
        killJob(jobClient);
        throw e;
    }
    List<T> target = testRecords.subList(0, numBeforeSuccess);
    checkResultWithSemantic(externalContext.createSinkDataReader(sinkSettings), target, semantic);
    // Step 4: restart the Flink job with the savepoint
    final StreamExecutionEnvironment restartEnv = testEnv.createExecutionEnvironment(TestEnvironmentSettings.builder().setConnectorJarPaths(externalContext.getConnectorJarPaths()).setSavepointRestorePath(savepointPath).build());
    restartEnv.enableCheckpointing(50);
    DataStreamSource<T> restartSource = restartEnv.fromSource(new FromElementsSource<>(Boundedness.CONTINUOUS_UNBOUNDED, testRecords, testRecords.size()), WatermarkStrategy.noWatermarks(), "restartSource").setParallelism(1);
    DataStream<T> sinkStream = restartSource.returns(externalContext.getProducedType());
    tryCreateSink(sinkStream, externalContext, sinkSettings).setParallelism(afterParallelism);
    addCollectSink(restartSource);
    final JobClient restartJobClient = restartEnv.executeAsync("Restart Test");
    try {
        // Check the result
        checkResultWithSemantic(externalContext.createSinkDataReader(sinkSettings), testRecords, semantic);
    } finally {
        executorService.shutdown();
        killJob(restartJobClient);
        iterator.close();
    }
}