Examples with CountDownLatch java.util.concurrent.CountDownLatch used on opensource projects

Example 36 with CountDownLatch

use of java.util.concurrent.CountDownLatch in project flink by apache.

the class AsynchronousFileIOChannelTest method testClosedButAddRequestAndRegisterListenerRace.

@Test
public void testClosedButAddRequestAndRegisterListenerRace() throws Exception {
    // -- Config ----------------------------------------------------------
    final int numberOfRuns = 1024;
    // -- Setup -----------------------------------------------------------
    final IOManagerAsync ioManager = new IOManagerAsync();
    final ExecutorService executor = Executors.newFixedThreadPool(2);
    final RequestQueue<WriteRequest> requestQueue = new RequestQueue<WriteRequest>();
    @SuppressWarnings("unchecked") final RequestDoneCallback<Buffer> ioChannelCallback = mock(RequestDoneCallback.class);
    final TestNotificationListener listener = new TestNotificationListener();
    // -- The Test --------------------------------------------------------
    try {
        // Repeatedly close the channel and add a request.
        for (int i = 0; i < numberOfRuns; i++) {
            final TestAsyncFileIOChannel ioChannel = new TestAsyncFileIOChannel(ioManager.createChannel(), requestQueue, ioChannelCallback, true);
            final CountDownLatch sync = new CountDownLatch(2);
            final WriteRequest request = mock(WriteRequest.class);
            ioChannel.close();
            // Add request task
            Callable<Void> addRequestTask = new Callable<Void>() {

                @Override
                public Void call() throws Exception {
                    try {
                        ioChannel.addRequest(request);
                    } catch (Throwable expected) {
                    } finally {
                        sync.countDown();
                    }
                    return null;
                }
            };
            // Listener
            Callable<Void> registerListenerTask = new Callable<Void>() {

                @Override
                public Void call() throws Exception {
                    try {
                        while (true) {
                            int current = listener.getNumberOfNotifications();
                            if (ioChannel.registerAllRequestsProcessedListener(listener)) {
                                listener.waitForNotification(current);
                            } else if (ioChannel.isClosed()) {
                                break;
                            }
                        }
                    } finally {
                        sync.countDown();
                    }
                    return null;
                }
            };
            executor.submit(addRequestTask);
            executor.submit(registerListenerTask);
            if (!sync.await(2, TimeUnit.MINUTES)) {
                fail("Test failed due to a timeout. This indicates a deadlock due to the way" + "that listeners are registered/notified in the asynchronous file I/O" + "channel.");
            }
        }
    } finally {
        ioManager.shutdown();
        executor.shutdown();
    }
}

Example 37 with CountDownLatch

use of java.util.concurrent.CountDownLatch in project flink by apache.

the class BufferFileWriterFileSegmentReaderTest method testWriteRead.

@Test
public void testWriteRead() throws IOException, InterruptedException {
    int numBuffers = 1024;
    int currentNumber = 0;
    final int minBufferSize = BUFFER_SIZE / 4;
    // Write buffers filled with ascending numbers...
    for (int i = 0; i < numBuffers; i++) {
        final Buffer buffer = createBuffer();
        int size = getNextMultipleOf(getRandomNumberInRange(minBufferSize, BUFFER_SIZE), 4);
        buffer.setSize(size);
        currentNumber = fillBufferWithAscendingNumbers(buffer, currentNumber);
        writer.writeBlock(buffer);
    }
    // Make sure that the writes are finished
    writer.close();
    // Read buffers back in...
    for (int i = 0; i < numBuffers; i++) {
        assertFalse(reader.hasReachedEndOfFile());
        reader.read();
    }
    // Wait for all requests to be finished
    final CountDownLatch sync = new CountDownLatch(1);
    final NotificationListener listener = new NotificationListener() {

        @Override
        public void onNotification() {
            sync.countDown();
        }
    };
    if (reader.registerAllRequestsProcessedListener(listener)) {
        sync.await();
    }
    assertTrue(reader.hasReachedEndOfFile());
    // Verify that the content is the same
    assertEquals("Read less buffers than written.", numBuffers, returnedFileSegments.size());
    currentNumber = 0;
    FileSegment fileSegment;
    ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
    while ((fileSegment = returnedFileSegments.poll()) != null) {
        buffer.position(0);
        buffer.limit(fileSegment.getLength());
        fileSegment.getFileChannel().read(buffer, fileSegment.getPosition());
        currentNumber = verifyBufferFilledWithAscendingNumbers(new Buffer(MemorySegmentFactory.wrap(buffer.array()), BUFFER_RECYCLER), currentNumber, fileSegment.getLength());
    }
    reader.close();
}

Example 38 with CountDownLatch

use of java.util.concurrent.CountDownLatch in project flink by apache.

the class SpillableSubpartitionTest method testConcurrentFinishAndReleaseMemory.

/**
	 * Tests a fix for FLINK-2384.
	 *
	 * @see <a href="https://issues.apache.org/jira/browse/FLINK-2384">FLINK-2384</a>
	 */
@Test
public void testConcurrentFinishAndReleaseMemory() throws Exception {
    // Latches to blocking
    final CountDownLatch doneLatch = new CountDownLatch(1);
    final CountDownLatch blockLatch = new CountDownLatch(1);
    // Blocking spill writer (blocks on the close call)
    AsynchronousBufferFileWriter spillWriter = mock(AsynchronousBufferFileWriter.class);
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            blockLatch.countDown();
            doneLatch.await();
            return null;
        }
    }).when(spillWriter).close();
    // Mock I/O manager returning the blocking spill writer
    IOManager ioManager = mock(IOManager.class);
    when(ioManager.createBufferFileWriter(any(FileIOChannel.ID.class))).thenReturn(spillWriter);
    // The partition
    final SpillableSubpartition partition = new SpillableSubpartition(0, mock(ResultPartition.class), ioManager);
    // Spill the partition initially (creates the spill writer)
    assertEquals(0, partition.releaseMemory());
    ExecutorService executor = Executors.newSingleThreadExecutor();
    // Finish the partition (this blocks because of the mock blocking writer)
    Future<Void> blockingFinish = executor.submit(new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            partition.finish();
            return null;
        }
    });
    // Ensure that the blocking call has been made
    blockLatch.await();
    // This call needs to go through. FLINK-2384 discovered a bug, in
    // which the finish call was holding a lock, which was leading to a
    // deadlock when another operation on the partition was happening.
    partition.releaseMemory();
    // Check that the finish call succeeded w/o problems as well to avoid
    // false test successes.
    doneLatch.countDown();
    blockingFinish.get();
}

Example 39 with CountDownLatch

use of java.util.concurrent.CountDownLatch in project flink by apache.

the class AkkaRpcServiceTest method testScheduledExecutorServicePeriodicSchedule.

/**
	 * Tests that the RPC service's scheduled executor service can execute runnables at a fixed
	 * rate.
	 */
@Test(timeout = 60000)
public void testScheduledExecutorServicePeriodicSchedule() throws ExecutionException, InterruptedException {
    ScheduledExecutor scheduledExecutor = akkaRpcService.getScheduledExecutor();
    final int tries = 4;
    final long delay = 10L;
    final CountDownLatch countDownLatch = new CountDownLatch(tries);
    long currentTime = System.nanoTime();
    ScheduledFuture<?> future = scheduledExecutor.scheduleAtFixedRate(new Runnable() {

        @Override
        public void run() {
            countDownLatch.countDown();
        }
    }, delay, delay, TimeUnit.MILLISECONDS);
    assertTrue(!future.isDone());
    countDownLatch.await();
    // the future should not complete since we have a periodic task
    assertTrue(!future.isDone());
    long finalTime = System.nanoTime() - currentTime;
    // the processing should have taken at least delay times the number of count downs.
    assertTrue(finalTime >= tries * delay);
    future.cancel(true);
}

Example 40 with CountDownLatch

use of java.util.concurrent.CountDownLatch in project flink by apache.

the class JobManagerHACheckpointRecoveryITCase method testCheckpointedStreamingSumProgram.

/**
	 * Simple checkpointed streaming sum.
	 *
	 * <p>The sources (Parallelism) count until sequenceEnd. The sink (1) sums up all counts and
	 * returns it to the main thread via a static variable. We wait until some checkpoints are
	 * completed and sanity check that the sources recover with an updated state to make sure that
	 * this test actually tests something.
	 */
@Test
@RetryOnFailure(times = 1)
public void testCheckpointedStreamingSumProgram() throws Exception {
    // Config
    final int checkpointingInterval = 200;
    final int sequenceEnd = 5000;
    final long expectedSum = Parallelism * sequenceEnd * (sequenceEnd + 1) / 2;
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(Parallelism);
    env.enableCheckpointing(checkpointingInterval);
    env.addSource(new CheckpointedSequenceSource(sequenceEnd)).addSink(new CountingSink()).setParallelism(1);
    JobGraph jobGraph = env.getStreamGraph().getJobGraph();
    Configuration config = ZooKeeperTestUtils.createZooKeeperHAConfig(ZooKeeper.getConnectString(), FileStateBackendBasePath.getAbsoluteFile().toURI().toString());
    config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, Parallelism);
    ActorSystem testSystem = null;
    final JobManagerProcess[] jobManagerProcess = new JobManagerProcess[2];
    LeaderRetrievalService leaderRetrievalService = null;
    ActorSystem taskManagerSystem = null;
    try {
        final Deadline deadline = TestTimeOut.fromNow();
        // Test actor system
        testSystem = AkkaUtils.createActorSystem(new Configuration(), new Some<>(new Tuple2<String, Object>("localhost", 0)));
        // The job managers
        jobManagerProcess[0] = new JobManagerProcess(0, config);
        jobManagerProcess[1] = new JobManagerProcess(1, config);
        jobManagerProcess[0].startProcess();
        jobManagerProcess[1].startProcess();
        // Leader listener
        TestingListener leaderListener = new TestingListener();
        leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(config);
        leaderRetrievalService.start(leaderListener);
        // The task manager
        taskManagerSystem = AkkaUtils.createActorSystem(config, Option.apply(new Tuple2<String, Object>("localhost", 0)));
        TaskManager.startTaskManagerComponentsAndActor(config, ResourceID.generate(), taskManagerSystem, "localhost", Option.<String>empty(), Option.<LeaderRetrievalService>empty(), false, TaskManager.class);
        {
            // Initial submission
            leaderListener.waitForNewLeader(deadline.timeLeft().toMillis());
            String leaderAddress = leaderListener.getAddress();
            UUID leaderId = leaderListener.getLeaderSessionID();
            // Get the leader ref
            ActorRef leaderRef = AkkaUtils.getActorRef(leaderAddress, testSystem, deadline.timeLeft());
            ActorGateway leader = new AkkaActorGateway(leaderRef, leaderId);
            // Submit the job in detached mode
            leader.tell(new SubmitJob(jobGraph, ListeningBehaviour.DETACHED));
            JobManagerActorTestUtils.waitForJobStatus(jobGraph.getJobID(), JobStatus.RUNNING, leader, deadline.timeLeft());
        }
        // Who's the boss?
        JobManagerProcess leadingJobManagerProcess;
        if (jobManagerProcess[0].getJobManagerAkkaURL(deadline.timeLeft()).equals(leaderListener.getAddress())) {
            leadingJobManagerProcess = jobManagerProcess[0];
        } else {
            leadingJobManagerProcess = jobManagerProcess[1];
        }
        CompletedCheckpointsLatch.await();
        // Kill the leading job manager process
        leadingJobManagerProcess.destroy();
        {
            // Recovery by the standby JobManager
            leaderListener.waitForNewLeader(deadline.timeLeft().toMillis());
            String leaderAddress = leaderListener.getAddress();
            UUID leaderId = leaderListener.getLeaderSessionID();
            ActorRef leaderRef = AkkaUtils.getActorRef(leaderAddress, testSystem, deadline.timeLeft());
            ActorGateway leader = new AkkaActorGateway(leaderRef, leaderId);
            JobManagerActorTestUtils.waitForJobStatus(jobGraph.getJobID(), JobStatus.RUNNING, leader, deadline.timeLeft());
        }
        // Wait to finish
        FinalCountLatch.await();
        assertEquals(expectedSum, (long) FinalCount.get());
        for (int i = 0; i < Parallelism; i++) {
            assertNotEquals(0, RecoveredStates.get(i));
        }
    } catch (Throwable t) {
        // Reset all static state for test retries
        CompletedCheckpointsLatch = new CountDownLatch(2);
        RecoveredStates = new AtomicLongArray(Parallelism);
        FinalCountLatch = new CountDownLatch(1);
        FinalCount = new AtomicReference<>();
        LastElement = -1;
        // Print early (in some situations the process logs get too big
        // for Travis and the root problem is not shown)
        t.printStackTrace();
        // In case of an error, print the job manager process logs.
        if (jobManagerProcess[0] != null) {
            jobManagerProcess[0].printProcessLog();
        }
        if (jobManagerProcess[1] != null) {
            jobManagerProcess[1].printProcessLog();
        }
        throw t;
    } finally {
        if (jobManagerProcess[0] != null) {
            jobManagerProcess[0].destroy();
        }
        if (jobManagerProcess[1] != null) {
            jobManagerProcess[1].destroy();
        }
        if (leaderRetrievalService != null) {
            leaderRetrievalService.stop();
        }
        if (taskManagerSystem != null) {
            taskManagerSystem.shutdown();
        }
        if (testSystem != null) {
            testSystem.shutdown();
        }
    }
}