Examples with CheckedThread org.apache.flink.core.testutils.CheckedThread used on opensource projects

Example 6 with CheckedThread

use of org.apache.flink.core.testutils.CheckedThread in project flink by apache.

the class SafetyNetCloseableRegistryTest method testCorrectScopesForSafetyNet.

@Test
public void testCorrectScopesForSafetyNet() throws Exception {
    CheckedThread t1 = new CheckedThread() {

        @Override
        public void go() throws Exception {
            try {
                FileSystem fs1 = FileSystem.getLocalFileSystem();
                // ensure no safety net in place
                Assert.assertFalse(fs1 instanceof SafetyNetWrapperFileSystem);
                FileSystemSafetyNet.initializeSafetyNetForThread();
                fs1 = FileSystem.getLocalFileSystem();
                // ensure safety net is in place now
                Assert.assertTrue(fs1 instanceof SafetyNetWrapperFileSystem);
                Path tmp = new Path(tmpFolder.newFolder().toURI().toString(), "test_file");
                try (FSDataOutputStream stream = fs1.create(tmp, FileSystem.WriteMode.NO_OVERWRITE)) {
                    CheckedThread t2 = new CheckedThread() {

                        @Override
                        public void go() {
                            FileSystem fs2 = FileSystem.getLocalFileSystem();
                            // ensure the safety net does not leak here
                            Assert.assertFalse(fs2 instanceof SafetyNetWrapperFileSystem);
                            FileSystemSafetyNet.initializeSafetyNetForThread();
                            fs2 = FileSystem.getLocalFileSystem();
                            // ensure we can bring another safety net in place
                            Assert.assertTrue(fs2 instanceof SafetyNetWrapperFileSystem);
                            FileSystemSafetyNet.closeSafetyNetAndGuardedResourcesForThread();
                            fs2 = FileSystem.getLocalFileSystem();
                            // and that we can remove it again
                            Assert.assertFalse(fs2 instanceof SafetyNetWrapperFileSystem);
                        }
                    };
                    t2.start();
                    t2.sync();
                    // ensure stream is still open and was never closed by any
                    // interferences
                    stream.write(42);
                    FileSystemSafetyNet.closeSafetyNetAndGuardedResourcesForThread();
                    // ensure leaking stream was closed
                    try {
                        stream.write(43);
                        Assert.fail();
                    } catch (IOException ignore) {
                    }
                    fs1 = FileSystem.getLocalFileSystem();
                    // ensure safety net was removed
                    Assert.assertFalse(fs1 instanceof SafetyNetWrapperFileSystem);
                } finally {
                    fs1.delete(tmp, false);
                }
            } catch (Exception e) {
                Assert.fail(ExceptionUtils.stringifyException(e));
            }
        }
    };
    t1.start();
    t1.sync();
}

Example 7 with CheckedThread

use of org.apache.flink.core.testutils.CheckedThread in project flink by apache.

the class AvroSerializerConcurrencyTest method testConcurrentUseOfSerializer.

@Test
public void testConcurrentUseOfSerializer() throws Exception {
    final AvroSerializer<String> serializer = new AvroSerializer<>(String.class);
    final BlockerSync sync = new BlockerSync();
    final DataOutputView regularOut = new DataOutputSerializer(32);
    final DataOutputView lockingOut = new LockingView(sync);
    // this thread serializes and gets stuck there
    final CheckedThread thread = new CheckedThread("serializer") {

        @Override
        public void go() throws Exception {
            serializer.serialize("a value", lockingOut);
        }
    };
    thread.start();
    sync.awaitBlocker();
    // this should fail with an exception
    try {
        serializer.serialize("value", regularOut);
        fail("should have failed with an exception");
    } catch (IllegalStateException e) {
    // expected
    } finally {
        // release the thread that serializes
        sync.releaseBlocker();
    }
    // this propagates exceptions from the spawned thread
    thread.sync();
}

Example 8 with CheckedThread

use of org.apache.flink.core.testutils.CheckedThread in project flink by apache.

the class CassandraSinkBaseTest method testWaitForPendingUpdatesOnClose.

@Test(timeout = DEFAULT_TEST_TIMEOUT)
public void testWaitForPendingUpdatesOnClose() throws Exception {
    TestCassandraSink casSinkFunc = new TestCassandraSink();
    try (OneInputStreamOperatorTestHarness<String, Object> testHarness = createOpenedTestHarness(casSinkFunc)) {
        CompletableFuture<ResultSet> completableFuture = new CompletableFuture<>();
        casSinkFunc.enqueueCompletableFuture(completableFuture);
        casSinkFunc.invoke("hello");
        Assert.assertEquals(1, casSinkFunc.getAcquiredPermits());
        final CountDownLatch latch = new CountDownLatch(1);
        Thread t = new CheckedThread("Flink-CassandraSinkBaseTest") {

            @Override
            public void go() throws Exception {
                testHarness.close();
                latch.countDown();
            }
        };
        t.start();
        while (t.getState() != Thread.State.TIMED_WAITING) {
            Thread.sleep(5);
        }
        Assert.assertEquals(1, casSinkFunc.getAcquiredPermits());
        completableFuture.complete(null);
        latch.await();
        Assert.assertEquals(0, casSinkFunc.getAcquiredPermits());
    }
}

Example 9 with CheckedThread

use of org.apache.flink.core.testutils.CheckedThread in project flink by apache.

the class ElasticsearchSinkBaseTest method testItemFailureRethrownOnCheckpointAfterFlush.

/**
 * Tests that any item failure in the listener callbacks due to flushing on an immediately
 * following checkpoint is rethrown; we set a timeout because the test will not finish if the
 * logic is broken.
 */
@Test(timeout = 5000)
public void testItemFailureRethrownOnCheckpointAfterFlush() throws Throwable {
    final DummyElasticsearchSink<String> sink = new DummyElasticsearchSink<>(new HashMap<String, String>(), new SimpleSinkFunction<String>(), new NoOpFailureHandler());
    final OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink));
    testHarness.open();
    // setup the next bulk request, and its mock item failures
    List<Exception> mockResponsesList = new ArrayList<>(2);
    // the first request in a bulk will succeed
    mockResponsesList.add(null);
    mockResponsesList.add(new Exception(// the second request in a bulk will fail
    "artificial failure for record"));
    sink.setMockItemFailuresListForNextBulkItemResponses(mockResponsesList);
    testHarness.processElement(new StreamRecord<>("msg-1"));
    verify(sink.getMockBulkProcessor(), times(1)).add(any(IndexRequest.class));
    // manually execute the next bulk request (1 request only, thus should succeed)
    sink.manualBulkRequestWithAllPendingRequests();
    // setup the requests to be flushed in the snapshot
    testHarness.processElement(new StreamRecord<>("msg-2"));
    testHarness.processElement(new StreamRecord<>("msg-3"));
    verify(sink.getMockBulkProcessor(), times(3)).add(any(IndexRequest.class));
    CheckedThread snapshotThread = new CheckedThread() {

        @Override
        public void go() throws Exception {
            testHarness.snapshot(1L, 1000L);
        }
    };
    snapshotThread.start();
    // the snapshot should eventually be blocked before snapshot triggers flushing
    while (snapshotThread.getState() != Thread.State.WAITING) {
        Thread.sleep(10);
    }
    // let the snapshot-triggered flush continue (2 records in the bulk, so the 2nd one should
    // fail)
    sink.continueFlush();
    try {
        snapshotThread.sync();
    } catch (Exception e) {
        // the snapshot should have failed with the failure from the 2nd request
        Assert.assertTrue(e.getCause().getCause().getMessage().contains("artificial failure for record"));
        // test succeeded
        return;
    }
    Assert.fail();
}

Example 10 with CheckedThread

use of org.apache.flink.core.testutils.CheckedThread in project flink by apache.

the class ElasticsearchSinkBaseTest method testBulkFailureRethrownOnOnCheckpointAfterFlush.

/**
 * Tests that any bulk failure in the listener callbacks due to flushing on an immediately
 * following checkpoint is rethrown; we set a timeout because the test will not finish if the
 * logic is broken.
 */
@Test(timeout = 5000)
public void testBulkFailureRethrownOnOnCheckpointAfterFlush() throws Throwable {
    final DummyElasticsearchSink<String> sink = new DummyElasticsearchSink<>(new HashMap<String, String>(), new SimpleSinkFunction<String>(), new NoOpFailureHandler());
    final OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink));
    testHarness.open();
    // setup the next bulk request, and let bulk request succeed
    sink.setMockItemFailuresListForNextBulkItemResponses(Collections.singletonList((Exception) null));
    testHarness.processElement(new StreamRecord<>("msg-1"));
    verify(sink.getMockBulkProcessor(), times(1)).add(any(IndexRequest.class));
    // manually execute the next bulk request
    sink.manualBulkRequestWithAllPendingRequests();
    // setup the requests to be flushed in the snapshot
    testHarness.processElement(new StreamRecord<>("msg-2"));
    testHarness.processElement(new StreamRecord<>("msg-3"));
    verify(sink.getMockBulkProcessor(), times(3)).add(any(IndexRequest.class));
    CheckedThread snapshotThread = new CheckedThread() {

        @Override
        public void go() throws Exception {
            testHarness.snapshot(1L, 1000L);
        }
    };
    snapshotThread.start();
    // the snapshot should eventually be blocked before snapshot triggers flushing
    while (snapshotThread.getState() != Thread.State.WAITING) {
        Thread.sleep(10);
    }
    // for the snapshot-triggered flush, we let the bulk request fail completely
    sink.setFailNextBulkRequestCompletely(new Exception("artificial failure for bulk request"));
    // let the snapshot-triggered flush continue (bulk request should fail completely)
    sink.continueFlush();
    try {
        snapshotThread.sync();
    } catch (Exception e) {
        // the snapshot should have failed with the bulk request failure
        Assert.assertTrue(e.getCause().getCause().getMessage().contains("artificial failure for bulk request"));
        // test succeeded
        return;
    }
    Assert.fail();
}