Examples with CheckedThread org.apache.flink.core.testutils.CheckedThread used on opensource projects

Example 41 with CheckedThread

use of org.apache.flink.core.testutils.CheckedThread in project flink by apache.

the class BoundedBlockingSubpartitionWriteReadTest method testRead10ConsumersConcurrent.

@Test
public void testRead10ConsumersConcurrent() throws Exception {
    final int numLongs = 15_000_000;
    // setup
    final BoundedBlockingSubpartition subpartition = createAndFillPartition(numLongs);
    // test
    final LongReader[] readerThreads = createSubpartitionLongReaders(subpartition, 10, numLongs, subpartition.getBuffersInBacklogUnsafe(), compressionEnabled);
    for (CheckedThread t : readerThreads) {
        t.start();
    }
    // check
    for (CheckedThread t : readerThreads) {
        // this propagates assertion errors out from the threads
        t.sync();
    }
    // cleanup
    subpartition.release();
}

Example 42 with CheckedThread

use of org.apache.flink.core.testutils.CheckedThread in project flink by apache.

the class ElasticsearchSinkBaseTest method testAtLeastOnceSink.

/**
 * Tests that the sink correctly waits for pending requests (including re-added requests) on
 * checkpoints; we set a timeout because the test will not finish if the logic is broken.
 */
@Test(timeout = 5000)
public void testAtLeastOnceSink() throws Throwable {
    final DummyElasticsearchSink<String> sink = new DummyElasticsearchSink<>(new HashMap<String, String>(), new SimpleSinkFunction<String>(), // use a failure handler that simply
    new DummyRetryFailureHandler());
    // re-adds requests
    final OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink));
    testHarness.open();
    // setup the next bulk request, and its mock item failures;
    // it contains 1 request, which will fail and re-added to the next bulk request
    sink.setMockItemFailuresListForNextBulkItemResponses(Collections.singletonList(new Exception("artificial failure for record")));
    testHarness.processElement(new StreamRecord<>("msg"));
    verify(sink.getMockBulkProcessor(), times(1)).add(any(IndexRequest.class));
    CheckedThread snapshotThread = new CheckedThread() {

        @Override
        public void go() throws Exception {
            testHarness.snapshot(1L, 1000L);
        }
    };
    snapshotThread.start();
    // the snapshot should eventually be blocked before snapshot triggers flushing
    while (snapshotThread.getState() != Thread.State.WAITING) {
        Thread.sleep(10);
    }
    sink.continueFlush();
    // flush
    while (snapshotThread.getState() != Thread.State.WAITING) {
        Thread.sleep(10);
    }
    // current number of pending request should be 1 due to the re-add
    Assert.assertEquals(1, sink.getNumPendingRequests());
    // this time, let the bulk request succeed, so no-more requests are re-added
    sink.setMockItemFailuresListForNextBulkItemResponses(Collections.singletonList((Exception) null));
    sink.continueFlush();
    // the snapshot should finish with no exceptions
    snapshotThread.sync();
    testHarness.close();
}

Example 43 with CheckedThread

use of org.apache.flink.core.testutils.CheckedThread in project flink by apache.

the class JobManagerMetricsITCase method setUp.

@Before
public void setUp() throws Exception {
    jobExecuteThread = new CheckedThread() {

        @Override
        public void go() throws Exception {
            StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
            env.addSource(new SourceFunction<String>() {

                @Override
                public void run(SourceContext<String> ctx) throws Exception {
                    sync.block();
                }

                @Override
                public void cancel() {
                    sync.releaseBlocker();
                }
            }).addSink(new PrintSinkFunction());
            env.execute();
        }
    };
    jobExecuteThread.start();
    sync.awaitBlocker();
}

Example 44 with CheckedThread

use of org.apache.flink.core.testutils.CheckedThread in project flink by apache.

the class AccumulatorLiveITCase method submitJobAndVerifyResults.

private static void submitJobAndVerifyResults(JobGraph jobGraph) throws Exception {
    Deadline deadline = Deadline.now().plus(Duration.ofSeconds(30));
    final ClusterClient<?> client = MINI_CLUSTER_RESOURCE.getClusterClient();
    final CheckedThread submissionThread = new CheckedThread() {

        @Override
        public void go() throws Exception {
            submitJobAndWaitForResult(client, jobGraph, getClass().getClassLoader());
        }
    };
    submissionThread.start();
    try {
        NotifyingMapper.notifyLatch.await();
        // verify using the ClusterClient
        verifyResults(jobGraph, deadline, client);
        // verify using the MiniClusterJobClient
        verifyResults(jobGraph, deadline, null);
        NotifyingMapper.shutdownLatch.trigger();
    } finally {
        NotifyingMapper.shutdownLatch.trigger();
        // wait for the job to have terminated
        submissionThread.sync();
    }
}

Example 45 with CheckedThread

use of org.apache.flink.core.testutils.CheckedThread in project flink by apache.

the class JobRetrievalITCase method testJobRetrieval.

@Test
public void testJobRetrieval() throws Exception {
    final JobVertex imalock = new JobVertex("imalock");
    imalock.setInvokableClass(SemaphoreInvokable.class);
    imalock.setParallelism(1);
    final JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(imalock);
    final JobID jobId = jobGraph.getJobID();
    // acquire the lock to make sure that the job cannot complete until the job client
    // has been attached in resumingThread
    lock.acquire();
    client.submitJob(jobGraph).get();
    final CheckedThread resumingThread = new CheckedThread("Flink-Job-Retriever") {

        @Override
        public void go() throws Exception {
            assertNotNull(client.requestJobResult(jobId).get());
        }
    };
    // wait until the job is running
    while (client.listJobs().get().isEmpty()) {
        Thread.sleep(50);
    }
    // kick off resuming
    resumingThread.start();
    // wait for client to connect
    while (resumingThread.getState() != Thread.State.WAITING) {
        Thread.sleep(10);
    }
    // client has connected, we can release the lock
    lock.release();
    resumingThread.sync();
}