Examples with CheckedThread org.apache.flink.core.testutils.CheckedThread used on opensource projects

Example 26 with CheckedThread

use of org.apache.flink.core.testutils.CheckedThread in project flink by apache.

the class FileUtilsTest method testDeleteDirectoryConcurrently.

@Test
public void testDeleteDirectoryConcurrently() throws Exception {
    final File parent = tmp.newFolder();
    generateRandomDirs(parent, 20, 5, 3);
    // start three concurrent threads that delete the contents
    CheckedThread t1 = new Deleter(parent);
    CheckedThread t2 = new Deleter(parent);
    CheckedThread t3 = new Deleter(parent);
    t1.start();
    t2.start();
    t3.start();
    t1.sync();
    t2.sync();
    t3.sync();
    // assert is empty
    assertFalse(parent.exists());
}

Example 27 with CheckedThread

use of org.apache.flink.core.testutils.CheckedThread in project flink by apache.

the class FlinkSecurityManagerTest method testInheritedThrowUserExit.

@Test
public void testInheritedThrowUserExit() throws Exception {
    FlinkSecurityManager flinkSecurityManager = new FlinkSecurityManager(ClusterOptions.UserSystemExitMode.THROW, false);
    flinkSecurityManager.monitorUserSystemExit();
    try {
        flinkSecurityManager.checkExit(TEST_EXIT_CODE);
        fail();
    } catch (UserSystemExitException ignored) {
    }
    CheckedThread thread = new CheckedThread() {

        @Override
        public void go() {
            try {
                flinkSecurityManager.checkExit(TEST_EXIT_CODE);
                fail();
            } catch (UserSystemExitException ignored) {
            } catch (Throwable t) {
                fail();
            }
        }
    };
    thread.start();
    thread.sync();
}

Example 28 with CheckedThread

use of org.apache.flink.core.testutils.CheckedThread in project flink by apache.

the class FlinkKinesisProducerTest method testBackpressure.

/**
 * Test ensuring that the producer blocks if the queue limit is exceeded, until the queue length
 * drops below the limit; we set a timeout because the test will not finish if the logic is
 * broken.
 */
@Test(timeout = 10000)
public void testBackpressure() throws Throwable {
    final Deadline deadline = Deadline.fromNow(Duration.ofSeconds(10));
    final DummyFlinkKinesisProducer<String> producer = new DummyFlinkKinesisProducer<>(new SimpleStringSchema());
    producer.setQueueLimit(1);
    OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(producer));
    testHarness.open();
    UserRecordResult result = mock(UserRecordResult.class);
    when(result.isSuccessful()).thenReturn(true);
    CheckedThread msg1 = new CheckedThread() {

        @Override
        public void go() throws Exception {
            testHarness.processElement(new StreamRecord<>("msg-1"));
        }
    };
    msg1.start();
    msg1.trySync(deadline.timeLeftIfAny().toMillis());
    assertFalse("Flush triggered before reaching queue limit", msg1.isAlive());
    // consume msg-1 so that queue is empty again
    producer.getPendingRecordFutures().get(0).set(result);
    CheckedThread msg2 = new CheckedThread() {

        @Override
        public void go() throws Exception {
            testHarness.processElement(new StreamRecord<>("msg-2"));
        }
    };
    msg2.start();
    msg2.trySync(deadline.timeLeftIfAny().toMillis());
    assertFalse("Flush triggered before reaching queue limit", msg2.isAlive());
    CheckedThread moreElementsThread = new CheckedThread() {

        @Override
        public void go() throws Exception {
            // this should block until msg-2 is consumed
            testHarness.processElement(new StreamRecord<>("msg-3"));
            // this should block until msg-3 is consumed
            testHarness.processElement(new StreamRecord<>("msg-4"));
        }
    };
    moreElementsThread.start();
    assertTrue("Producer should still block, but doesn't", moreElementsThread.isAlive());
    // consume msg-2 from the queue, leaving msg-3 in the queue and msg-4 blocked
    while (producer.getPendingRecordFutures().size() < 2) {
        Thread.sleep(50);
    }
    producer.getPendingRecordFutures().get(1).set(result);
    assertTrue("Producer should still block, but doesn't", moreElementsThread.isAlive());
    // consume msg-3, blocked msg-4 can be inserted into the queue and block is released
    while (producer.getPendingRecordFutures().size() < 3) {
        Thread.sleep(50);
    }
    producer.getPendingRecordFutures().get(2).set(result);
    moreElementsThread.trySync(deadline.timeLeftIfAny().toMillis());
    assertFalse("Prodcuer still blocks although the queue is flushed", moreElementsThread.isAlive());
    producer.getPendingRecordFutures().get(3).set(result);
    testHarness.close();
}

Example 29 with CheckedThread

use of org.apache.flink.core.testutils.CheckedThread in project flink by apache.

the class KinesisDataFetcherTest method testCancelDuringDiscovery.

@Test
public void testCancelDuringDiscovery() throws Exception {
    final String stream = "fakeStream";
    final int numShards = 3;
    Properties standardProperties = TestUtils.getStandardProperties();
    standardProperties.setProperty(SHARD_DISCOVERY_INTERVAL_MILLIS, "10000000");
    final LinkedList<KinesisStreamShardState> testShardStates = new LinkedList<>();
    final TestSourceContext<String> sourceContext = new TestSourceContext<>();
    TestableKinesisDataFetcher<String> fetcher = new TestableKinesisDataFetcher<String>(singletonList(stream), sourceContext, standardProperties, new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()), 1, 0, new AtomicReference<>(), testShardStates, new HashMap<>(), FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(Collections.singletonMap(stream, numShards)));
    // FlinkKinesisConsumer is responsible for setting up the fetcher before it can be run;
    // run the consumer until it reaches the point where the fetcher starts to run
    final DummyFlinkKinesisConsumer<String> consumer = new DummyFlinkKinesisConsumer<>(TestUtils.getStandardProperties(), fetcher, 1, 0);
    CheckedThread consumerThread = new CheckedThread() {

        @Override
        public void go() throws Exception {
            consumer.run(new TestSourceContext<>());
        }
    };
    consumerThread.start();
    // wait for the second discovery to be triggered, that has a high probability to be inside
    // discovery sleep (10k s)
    fetcher.waitUntilDiscovery(2);
    Thread.sleep(1000);
    consumer.cancel();
    consumerThread.sync();
}

Example 30 with CheckedThread

use of org.apache.flink.core.testutils.CheckedThread in project flink by apache.

the class KinesisDataFetcherTest method testStreamToLastSeenShardStateIsCorrectlySetWhenNewShardsFoundSinceRestoredCheckpoint.

@Test
public void testStreamToLastSeenShardStateIsCorrectlySetWhenNewShardsFoundSinceRestoredCheckpoint() throws Exception {
    List<String> fakeStreams = new LinkedList<>();
    fakeStreams.add("fakeStream1");
    fakeStreams.add("fakeStream2");
    Map<StreamShardHandle, String> restoredStateUnderTest = new HashMap<>();
    // fakeStream1 has 3 shards before restore
    restoredStateUnderTest.put(new StreamShardHandle("fakeStream1", new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(0))), UUID.randomUUID().toString());
    restoredStateUnderTest.put(new StreamShardHandle("fakeStream1", new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(1))), UUID.randomUUID().toString());
    restoredStateUnderTest.put(new StreamShardHandle("fakeStream1", new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(2))), UUID.randomUUID().toString());
    // fakeStream2 has 2 shards before restore
    restoredStateUnderTest.put(new StreamShardHandle("fakeStream2", new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(0))), UUID.randomUUID().toString());
    restoredStateUnderTest.put(new StreamShardHandle("fakeStream2", new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(1))), UUID.randomUUID().toString());
    Map<String, Integer> streamToShardCount = new HashMap<>();
    streamToShardCount.put("fakeStream1", // fakeStream1 had 3 shards before & 1 new shard after restore
    3 + 1);
    streamToShardCount.put("fakeStream2", // fakeStream2 had 2 shards before & 3 new shard after restore
    2 + 3);
    HashMap<String, String> subscribedStreamsToLastSeenShardIdsUnderTest = KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams);
    // using a non-resharded streams kinesis behaviour to represent that Kinesis is not
    // resharded AFTER the restore
    final TestableKinesisDataFetcher<String> fetcher = new TestableKinesisDataFetcher<>(fakeStreams, new TestSourceContext<>(), TestUtils.getStandardProperties(), new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()), 10, 2, new AtomicReference<>(), new LinkedList<>(), subscribedStreamsToLastSeenShardIdsUnderTest, FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
    for (Map.Entry<StreamShardHandle, String> restoredState : restoredStateUnderTest.entrySet()) {
        fetcher.advanceLastDiscoveredShardOfStream(restoredState.getKey().getStreamName(), restoredState.getKey().getShard().getShardId());
        fetcher.registerNewSubscribedShardState(new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(restoredState.getKey()), restoredState.getKey(), new SequenceNumber(restoredState.getValue())));
    }
    CheckedThread runFetcherThread = new CheckedThread() {

        @Override
        public void go() throws Exception {
            fetcher.runFetcher();
        }
    };
    runFetcherThread.start();
    fetcher.waitUntilInitialDiscovery();
    fetcher.shutdownFetcher();
    runFetcherThread.sync();
    // assert that the streams tracked in the state are identical to the subscribed streams
    Set<String> streamsInState = subscribedStreamsToLastSeenShardIdsUnderTest.keySet();
    assertEquals(fakeStreams.size(), streamsInState.size());
    assertTrue(streamsInState.containsAll(fakeStreams));
    // assert that the last seen shards in state is correctly set
    for (Map.Entry<String, String> streamToLastSeenShard : subscribedStreamsToLastSeenShardIdsUnderTest.entrySet()) {
        assertEquals(KinesisShardIdGenerator.generateFromShardOrder(streamToShardCount.get(streamToLastSeenShard.getKey()) - 1), streamToLastSeenShard.getValue());
    }
}