Examples with OneShotLatch org.apache.flink.core.testutils.OneShotLatch used on opensource projects

Example 11 with OneShotLatch

use of org.apache.flink.core.testutils.OneShotLatch in project flink by apache.

the class StateBackendTestBase method testAsyncSnapshotCancellation.

@Test
public void testAsyncSnapshotCancellation() throws Exception {
    OneShotLatch blocker = new OneShotLatch();
    OneShotLatch waiter = new OneShotLatch();
    BlockerCheckpointStreamFactory streamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
    streamFactory.setWaiterLatch(waiter);
    streamFactory.setBlockerLatch(blocker);
    streamFactory.setAfterNumberInvocations(100);
    AbstractKeyedStateBackend<Integer> backend = null;
    try {
        backend = createKeyedBackend(IntSerializer.INSTANCE);
        if (!backend.supportsAsynchronousSnapshots()) {
            return;
        }
        InternalValueState<VoidNamespace, Integer> valueState = backend.createValueState(VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>("test", IntSerializer.INSTANCE));
        valueState.setCurrentNamespace(VoidNamespace.INSTANCE);
        for (int i = 0; i < 10; ++i) {
            backend.setCurrentKey(i);
            valueState.update(i);
        }
        RunnableFuture<KeyGroupsStateHandle> snapshot = backend.snapshot(0L, 0L, streamFactory, CheckpointOptions.forFullCheckpoint());
        Thread runner = new Thread(snapshot);
        runner.start();
        // wait until the code reached some stream read
        waiter.await();
        // close the backend to see if the close is propagated to the stream
        backend.close();
        //unblock the stream so that it can run into the IOException
        blocker.trigger();
        //dispose the backend
        backend.dispose();
        runner.join();
        try {
            snapshot.get();
            fail("Close was not propagated.");
        } catch (ExecutionException ex) {
        //ignore
        }
    } finally {
        if (null != backend) {
            IOUtils.closeQuietly(backend);
            backend.dispose();
        }
    }
}

Example 12 with OneShotLatch

use of org.apache.flink.core.testutils.OneShotLatch in project flink by apache.

the class AkkaRpcServiceTest method testScheduleRunnable.

// ------------------------------------------------------------------------
//  tests
// ------------------------------------------------------------------------
@Test
public void testScheduleRunnable() throws Exception {
    final OneShotLatch latch = new OneShotLatch();
    final long delay = 100;
    final long start = System.nanoTime();
    ScheduledFuture<?> scheduledFuture = akkaRpcService.scheduleRunnable(new Runnable() {

        @Override
        public void run() {
            latch.trigger();
        }
    }, delay, TimeUnit.MILLISECONDS);
    scheduledFuture.get();
    assertTrue(latch.isTriggered());
    final long stop = System.nanoTime();
    assertTrue("call was not properly delayed", ((stop - start) / 1000000) >= delay);
}

Example 13 with OneShotLatch

use of org.apache.flink.core.testutils.OneShotLatch in project flink by apache.

the class AkkaRpcServiceTest method testScheduledExecutorServiceSimpleSchedule.

/**
	 * Tests a simple scheduled runnable being executed by the RPC services scheduled executor
	 * service.
	 */
@Test(timeout = 60000)
public void testScheduledExecutorServiceSimpleSchedule() throws ExecutionException, InterruptedException {
    ScheduledExecutor scheduledExecutor = akkaRpcService.getScheduledExecutor();
    final OneShotLatch latch = new OneShotLatch();
    ScheduledFuture<?> future = scheduledExecutor.schedule(new Runnable() {

        @Override
        public void run() {
            latch.trigger();
        }
    }, 10L, TimeUnit.MILLISECONDS);
    future.get();
    // once the future is completed, then the latch should have been triggered
    assertTrue(latch.isTriggered());
}

Example 14 with OneShotLatch

use of org.apache.flink.core.testutils.OneShotLatch in project flink by apache.

the class TaskTest method createQueuesAndActors.

@Before
public void createQueuesAndActors() {
    taskManagerMessages = new LinkedBlockingQueue<Object>();
    jobManagerMessages = new LinkedBlockingQueue<Object>();
    listenerMessages = new LinkedBlockingQueue<Object>();
    taskManagerGateway = new ForwardingActorGateway(taskManagerMessages);
    jobManagerGateway = new ForwardingActorGateway(jobManagerMessages);
    listenerGateway = new ForwardingActorGateway(listenerMessages);
    listener = new ActorGatewayTaskExecutionStateListener(listenerGateway);
    taskManagerConnection = new ActorGatewayTaskManagerActions(taskManagerGateway);
    awaitLatch = new OneShotLatch();
    triggerLatch = new OneShotLatch();
    cancelLatch = new OneShotLatch();
}

Example 15 with OneShotLatch

use of org.apache.flink.core.testutils.OneShotLatch in project flink by apache.

the class ContinuousFileProcessingRescalingTest method testReaderScalingDown.

@Test
public void testReaderScalingDown() throws Exception {
    // simulates the scenario of scaling down from 2 to 1 instances
    final OneShotLatch waitingLatch = new OneShotLatch();
    // create the first instance and let it process the first split till element 5
    final OneShotLatch triggerLatch1 = new OneShotLatch();
    BlockingFileInputFormat format1 = new BlockingFileInputFormat(triggerLatch1, waitingLatch, new Path("test"), 20, 5);
    FileInputSplit[] splits = format1.createInputSplits(2);
    OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness1 = getTestHarness(format1, 2, 0);
    testHarness1.open();
    testHarness1.processElement(new StreamRecord<>(getTimestampedSplit(0, splits[0])));
    // wait until its arrives to element 5
    if (!triggerLatch1.isTriggered()) {
        triggerLatch1.await();
    }
    // create the second instance and let it process the second split till element 15
    final OneShotLatch triggerLatch2 = new OneShotLatch();
    BlockingFileInputFormat format2 = new BlockingFileInputFormat(triggerLatch2, waitingLatch, new Path("test"), 20, 15);
    OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness2 = getTestHarness(format2, 2, 1);
    testHarness2.open();
    testHarness2.processElement(new StreamRecord<>(getTimestampedSplit(0, splits[1])));
    // wait until its arrives to element 15
    if (!triggerLatch2.isTriggered()) {
        triggerLatch2.await();
    }
    // 1) clear the outputs of the two previous instances so that
    // we can compare their newly produced outputs with the merged one
    testHarness1.getOutput().clear();
    testHarness2.getOutput().clear();
    // 2) and take the snapshots from the previous instances and merge them
    // into a new one which will be then used to initialize a third instance
    OperatorStateHandles mergedState = AbstractStreamOperatorTestHarness.repackageState(testHarness2.snapshot(0, 0), testHarness1.snapshot(0, 0));
    // create the third instance
    final OneShotLatch wLatch = new OneShotLatch();
    final OneShotLatch tLatch = new OneShotLatch();
    BlockingFileInputFormat format = new BlockingFileInputFormat(wLatch, tLatch, new Path("test"), 20, 5);
    OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness = getTestHarness(format, 1, 0);
    // initialize the state of the new operator with the constructed by
    // combining the partial states of the instances above.
    testHarness.initializeState(mergedState);
    testHarness.open();
    // now restart the waiting operators
    wLatch.trigger();
    tLatch.trigger();
    waitingLatch.trigger();
    // and wait for the processing to finish
    synchronized (testHarness1.getCheckpointLock()) {
        testHarness1.close();
    }
    synchronized (testHarness2.getCheckpointLock()) {
        testHarness2.close();
    }
    synchronized (testHarness.getCheckpointLock()) {
        testHarness.close();
    }
    Queue<Object> expectedResult = new ArrayDeque<>();
    putElementsInQ(expectedResult, testHarness1.getOutput());
    putElementsInQ(expectedResult, testHarness2.getOutput());
    Queue<Object> actualResult = new ArrayDeque<>();
    putElementsInQ(actualResult, testHarness.getOutput());
    Assert.assertEquals(20, actualResult.size());
    Assert.assertArrayEquals(expectedResult.toArray(), actualResult.toArray());
}