Examples with MockBaseSource org.apache.flink.connector.base.source.reader.mocks.MockBaseSource used on opensource projects

Example 1 with MockBaseSource

use of org.apache.flink.connector.base.source.reader.mocks.MockBaseSource in project flink by apache.

the class HybridSourceTest method testBoundedness.

@Test
public void testBoundedness() {
    HybridSource<Integer> source;
    source = HybridSource.builder(new MockBaseSource(1, 1, Boundedness.BOUNDED)).addSource(new MockBaseSource(1, 1, Boundedness.BOUNDED)).build();
    assertEquals(Boundedness.BOUNDED, source.getBoundedness());
    source = HybridSource.builder(new MockBaseSource(1, 1, Boundedness.BOUNDED)).addSource(new MockBaseSource(1, 1, Boundedness.CONTINUOUS_UNBOUNDED)).build();
    assertEquals(Boundedness.CONTINUOUS_UNBOUNDED, source.getBoundedness());
    try {
        HybridSource.builder(new MockBaseSource(1, 1, Boundedness.CONTINUOUS_UNBOUNDED)).addSource(new MockBaseSource(1, 1, Boundedness.CONTINUOUS_UNBOUNDED)).build();
        fail("expected exception");
    } catch (IllegalArgumentException e) {
    // boundedness check to fail
    }
}

Example 2 with MockBaseSource

use of org.apache.flink.connector.base.source.reader.mocks.MockBaseSource in project flink by apache.

the class HybridSourceTest method testBuilderWithEnumeratorSuperclass.

@Test
public void testBuilderWithEnumeratorSuperclass() {
    HybridSource.SourceFactory<Integer, Source<Integer, ?, ?>, MockSplitEnumerator> sourceFactory = (HybridSource.SourceFactory<Integer, Source<Integer, ?, ?>, MockSplitEnumerator>) context -> {
        MockSplitEnumerator enumerator = context.getPreviousEnumerator();
        return new MockBaseSource(1, 1, Boundedness.BOUNDED);
    };
    HybridSource<Integer> source = new HybridSource.HybridSourceBuilder<Integer, MockSplitEnumerator>().<ExtendedMockSplitEnumerator, Source<Integer, ?, ?>>addSource(new MockBaseSource(1, 1, Boundedness.BOUNDED)).addSource(sourceFactory, Boundedness.BOUNDED).build();
    assertNotNull(source);
}

Example 3 with MockBaseSource

use of org.apache.flink.connector.base.source.reader.mocks.MockBaseSource in project flink by apache.

the class CoordinatedSourceITCase method testMultipleSources.

@Test
public void testMultipleSources() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    MockBaseSource source1 = new MockBaseSource(2, 10, Boundedness.BOUNDED);
    MockBaseSource source2 = new MockBaseSource(2, 10, 20, Boundedness.BOUNDED);
    DataStream<Integer> stream1 = env.fromSource(source1, WatermarkStrategy.noWatermarks(), "TestingSource1");
    DataStream<Integer> stream2 = env.fromSource(source2, WatermarkStrategy.noWatermarks(), "TestingSource2");
    executeAndVerify(env, stream1.union(stream2), 40);
}

Example 4 with MockBaseSource

use of org.apache.flink.connector.base.source.reader.mocks.MockBaseSource in project flink by apache.

the class SourceMetricsITCase method testMetrics.

private void testMetrics(WatermarkStrategy<Integer> strategy, boolean hasTimestamps) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    int numSplits = Math.max(1, env.getParallelism() - 2);
    env.getConfig().setAutoWatermarkInterval(1L);
    int numRecordsPerSplit = 10;
    MockBaseSource source = new MockBaseSource(numSplits, numRecordsPerSplit, Boundedness.BOUNDED);
    // make sure all parallel instances have processed the same amount of records before
    // validating metrics
    SharedReference<CyclicBarrier> beforeBarrier = sharedObjects.add(new CyclicBarrier(numSplits + 1));
    SharedReference<CyclicBarrier> afterBarrier = sharedObjects.add(new CyclicBarrier(numSplits + 1));
    int stopAtRecord1 = 3;
    int stopAtRecord2 = numRecordsPerSplit - 1;
    DataStream<Integer> stream = env.fromSource(source, strategy, "MetricTestingSource").map(i -> {
        if (i % numRecordsPerSplit == stopAtRecord1 || i % numRecordsPerSplit == stopAtRecord2) {
            beforeBarrier.get().await();
            afterBarrier.get().await();
        }
        return i;
    });
    stream.addSink(new DiscardingSink<>());
    JobClient jobClient = env.executeAsync();
    final JobID jobId = jobClient.getJobID();
    beforeBarrier.get().await();
    assertSourceMetrics(jobId, reporter, stopAtRecord1 + 1, numRecordsPerSplit, env.getParallelism(), numSplits, hasTimestamps);
    afterBarrier.get().await();
    beforeBarrier.get().await();
    assertSourceMetrics(jobId, reporter, stopAtRecord2 + 1, numRecordsPerSplit, env.getParallelism(), numSplits, hasTimestamps);
    afterBarrier.get().await();
    jobClient.getJobExecutionResult().get();
}

Example 5 with MockBaseSource

use of org.apache.flink.connector.base.source.reader.mocks.MockBaseSource in project flink by apache.

the class HybridSourceITCase method sourceWithFixedSwitchPosition.

private Source sourceWithFixedSwitchPosition() {
    int numSplits = 2;
    int numRecordsPerSplit = EXPECTED_RESULT.size() / 4;
    return HybridSource.builder(new MockBaseSource(numSplits, numRecordsPerSplit, Boundedness.BOUNDED)).addSource(new MockBaseSource(numSplits, numRecordsPerSplit, 20, Boundedness.BOUNDED)).build();
}