Examples with TestNotificationListener org.apache.flink.runtime.io.network.util.TestNotificationListener used on opensource projects

Example 1 with TestNotificationListener

use of org.apache.flink.runtime.io.network.util.TestNotificationListener in project flink by apache.

the class AsynchronousBufferFileWriterTest method testConcurrentSubscribeAndHandleRequest.

@Test
public void testConcurrentSubscribeAndHandleRequest() throws Exception {
    final ExecutorService executor = Executors.newFixedThreadPool(2);
    final TestNotificationListener listener = new TestNotificationListener();
    final Callable<Boolean> subscriber = new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return writer.registerAllRequestsProcessedListener(listener);
        }
    };
    final Callable<Void> requestHandler = new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            handleRequest();
            return null;
        }
    };
    try {
        // Repeat this to provoke races
        for (int i = 0; i < 50000; i++) {
            listener.reset();
            addRequest();
            Future<Void> handleRequestFuture = executor.submit(requestHandler);
            Future<Boolean> subscribeFuture = executor.submit(subscriber);
            handleRequestFuture.get();
            try {
                if (subscribeFuture.get()) {
                    assertEquals("Race: Successfully subscribed, but was never notified.", 1, listener.getNumberOfNotifications());
                } else {
                    assertEquals("Race: Never subscribed successfully, but was notified.", 0, listener.getNumberOfNotifications());
                }
            } catch (Throwable t) {
                System.out.println(i);
                Assert.fail(t.getMessage());
            }
        }
    } finally {
        executor.shutdownNow();
    }
}

Example 2 with TestNotificationListener

use of org.apache.flink.runtime.io.network.util.TestNotificationListener in project flink by apache.

the class AsynchronousFileIOChannelTest method testClosedButAddRequestAndRegisterListenerRace.

@Test
public void testClosedButAddRequestAndRegisterListenerRace() throws Exception {
    // -- Config ----------------------------------------------------------
    final int numberOfRuns = 1024;
    // -- Setup -----------------------------------------------------------
    final ExecutorService executor = Executors.newFixedThreadPool(2);
    final RequestQueue<WriteRequest> requestQueue = new RequestQueue<WriteRequest>();
    @SuppressWarnings("unchecked") final RequestDoneCallback<Buffer> ioChannelCallback = mock(RequestDoneCallback.class);
    final TestNotificationListener listener = new TestNotificationListener();
    // -- The Test --------------------------------------------------------
    try (final IOManagerAsync ioManager = new IOManagerAsync()) {
        // Repeatedly close the channel and add a request.
        for (int i = 0; i < numberOfRuns; i++) {
            final TestAsyncFileIOChannel ioChannel = new TestAsyncFileIOChannel(ioManager.createChannel(), requestQueue, ioChannelCallback, true);
            final CountDownLatch sync = new CountDownLatch(2);
            final WriteRequest request = mock(WriteRequest.class);
            ioChannel.close();
            // Add request task
            Callable<Void> addRequestTask = new Callable<Void>() {

                @Override
                public Void call() throws Exception {
                    try {
                        ioChannel.addRequest(request);
                    } catch (Throwable expected) {
                    } finally {
                        sync.countDown();
                    }
                    return null;
                }
            };
            // Listener
            Callable<Void> registerListenerTask = new Callable<Void>() {

                @Override
                public Void call() throws Exception {
                    try {
                        while (true) {
                            int current = listener.getNumberOfNotifications();
                            if (ioChannel.registerAllRequestsProcessedListener(listener)) {
                                listener.waitForNotification(current);
                            } else if (ioChannel.isClosed()) {
                                break;
                            }
                        }
                    } finally {
                        sync.countDown();
                    }
                    return null;
                }
            };
            executor.submit(addRequestTask);
            executor.submit(registerListenerTask);
            if (!sync.await(2, TimeUnit.MINUTES)) {
                fail("Test failed due to a timeout. This indicates a deadlock due to the way" + "that listeners are registered/notified in the asynchronous file I/O" + "channel.");
            }
        }
    } finally {
        executor.shutdown();
    }
}

Example 3 with TestNotificationListener

use of org.apache.flink.runtime.io.network.util.TestNotificationListener in project flink by apache.

the class AsynchronousBufferFileWriterTest method testSubscribe.

@Test
public void testSubscribe() throws Exception {
    final TestNotificationListener listener = new TestNotificationListener();
    // Unsuccessful subscription, because no outstanding requests
    assertFalse("Allowed to subscribe w/o any outstanding requests.", writer.registerAllRequestsProcessedListener(listener));
    // Successful subscription
    addRequest();
    assertTrue("Didn't allow to subscribe.", writer.registerAllRequestsProcessedListener(listener));
    // Test notification
    handleRequest();
    assertEquals("Listener was not notified.", 1, listener.getNumberOfNotifications());
}

Example 4 with TestNotificationListener

use of org.apache.flink.runtime.io.network.util.TestNotificationListener in project flink by apache.

the class AsynchronousBufferFileWriterTest method testSubscribeAndClose.

@Test
public void testSubscribeAndClose() throws IOException, InterruptedException {
    final TestNotificationListener listener = new TestNotificationListener();
    final AtomicReference<Throwable> error = new AtomicReference<Throwable>();
    final CountDownLatch sync = new CountDownLatch(1);
    addRequest();
    addRequest();
    writer.registerAllRequestsProcessedListener(listener);
    final Thread asyncCloseThread = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                writer.close();
            } catch (Throwable t) {
                error.set(t);
            } finally {
                sync.countDown();
            }
        }
    });
    asyncCloseThread.start();
    handleRequest();
    handleRequest();
    sync.await();
    assertEquals("Listener was not notified.", 1, listener.getNumberOfNotifications());
}

Example 5 with TestNotificationListener

use of org.apache.flink.runtime.io.network.util.TestNotificationListener in project flink by apache.

the class AsynchronousFileIOChannelTest method testAllRequestsProcessedListenerNotification.

@Test
public void testAllRequestsProcessedListenerNotification() throws Exception {
    // -- Config ----------------------------------------------------------
    final int numberOfRuns = 10;
    final int numberOfRequests = 100;
    // -- Setup -----------------------------------------------------------
    final ExecutorService executor = Executors.newFixedThreadPool(3);
    final Random random = new Random();
    final RequestQueue<WriteRequest> requestQueue = new RequestQueue<WriteRequest>();
    final RequestDoneCallback<Buffer> ioChannelCallback = mock(RequestDoneCallback.class);
    final TestNotificationListener listener = new TestNotificationListener();
    // -- The Test --------------------------------------------------------
    try (final IOManagerAsync ioManager = new IOManagerAsync()) {
        for (int run = 0; run < numberOfRuns; run++) {
            final TestAsyncFileIOChannel ioChannel = new TestAsyncFileIOChannel(ioManager.createChannel(), requestQueue, ioChannelCallback, true);
            final CountDownLatch sync = new CountDownLatch(3);
            // The mock requests
            final Buffer buffer = mock(Buffer.class);
            final WriteRequest request = mock(WriteRequest.class);
            // Add requests task
            Callable<Void> addRequestsTask = new Callable<Void>() {

                @Override
                public Void call() throws Exception {
                    for (int i = 0; i < numberOfRuns; i++) {
                        LOG.debug("Starting run {}.", i + 1);
                        for (int j = 0; j < numberOfRequests; j++) {
                            ioChannel.addRequest(request);
                        }
                        LOG.debug("Added all ({}) requests of run {}.", numberOfRequests, i + 1);
                        int sleep = random.nextInt(10);
                        LOG.debug("Sleeping for {} ms before next run.", sleep);
                        Thread.sleep(sleep);
                    }
                    LOG.debug("Done. Closing channel.");
                    ioChannel.close();
                    sync.countDown();
                    return null;
                }
            };
            // Process requests task
            Callable<Void> processRequestsTask = new Callable<Void>() {

                @Override
                public Void call() throws Exception {
                    int total = numberOfRequests * numberOfRuns;
                    for (int i = 0; i < total; i++) {
                        requestQueue.take();
                        ioChannel.handleProcessedBuffer(buffer, null);
                    }
                    LOG.debug("Processed all ({}) requests.", numberOfRequests);
                    sync.countDown();
                    return null;
                }
            };
            // Listener
            Callable<Void> registerListenerTask = new Callable<Void>() {

                @Override
                public Void call() throws Exception {
                    while (true) {
                        int current = listener.getNumberOfNotifications();
                        if (ioChannel.registerAllRequestsProcessedListener(listener)) {
                            listener.waitForNotification(current);
                        } else if (ioChannel.isClosed()) {
                            break;
                        }
                    }
                    LOG.debug("Stopping listener. Channel closed.");
                    sync.countDown();
                    return null;
                }
            };
            // Run tasks in random order
            final List<Callable<?>> tasks = new LinkedList<Callable<?>>();
            tasks.add(addRequestsTask);
            tasks.add(processRequestsTask);
            tasks.add(registerListenerTask);
            Collections.shuffle(tasks);
            for (Callable<?> task : tasks) {
                executor.submit(task);
            }
            if (!sync.await(2, TimeUnit.MINUTES)) {
                fail("Test failed due to a timeout. This indicates a deadlock due to the way" + "that listeners are registered/notified in the asynchronous file I/O" + "channel.");
            }
            listener.reset();
        }
    } finally {
        executor.shutdown();
    }
}