Examples with MockEnvironment org.apache.flink.runtime.operators.testutils.MockEnvironment used on opensource projects

Example 21 with MockEnvironment

use of org.apache.flink.runtime.operators.testutils.MockEnvironment in project flink by apache.

the class StreamTaskTest method testFailingAsyncCheckpointRunnable.

/**
 * Tests that in case of a failing AsyncCheckpointRunnable all operator snapshot results are
 * cancelled and all non partitioned state handles are discarded.
 */
@Test
public void testFailingAsyncCheckpointRunnable() throws Exception {
    // mock the new state operator snapshots
    OperatorSnapshotFutures operatorSnapshotResult1 = mock(OperatorSnapshotFutures.class);
    OperatorSnapshotFutures operatorSnapshotResult2 = mock(OperatorSnapshotFutures.class);
    OperatorSnapshotFutures operatorSnapshotResult3 = mock(OperatorSnapshotFutures.class);
    RunnableFuture<SnapshotResult<OperatorStateHandle>> failingFuture = mock(RunnableFuture.class);
    when(failingFuture.get()).thenThrow(new ExecutionException(new Exception("Test exception")));
    when(operatorSnapshotResult3.getOperatorStateRawFuture()).thenReturn(failingFuture);
    try (MockEnvironment mockEnvironment = new MockEnvironmentBuilder().build()) {
        RunningTask<MockStreamTask> task = runTask(() -> createMockStreamTask(mockEnvironment, operatorChain(streamOperatorWithSnapshot(operatorSnapshotResult1), streamOperatorWithSnapshot(operatorSnapshotResult2), streamOperatorWithSnapshot(operatorSnapshotResult3))));
        MockStreamTask streamTask = task.streamTask;
        waitTaskIsRunning(streamTask, task.invocationFuture);
        mockEnvironment.setExpectedExternalFailureCause(Throwable.class);
        streamTask.triggerCheckpointAsync(new CheckpointMetaData(42L, 1L), CheckpointOptions.forCheckpointWithDefaultLocation()).get();
        // wait for the completion of the async task
        ExecutorService executor = streamTask.getAsyncOperationsThreadPool();
        executor.shutdown();
        if (!executor.awaitTermination(10000L, TimeUnit.MILLISECONDS)) {
            fail("Executor did not shut down within the given timeout. This indicates that the " + "checkpointing did not resume.");
        }
        assertTrue(mockEnvironment.getActualExternalFailureCause().isPresent());
        verify(operatorSnapshotResult1).cancel();
        verify(operatorSnapshotResult2).cancel();
        verify(operatorSnapshotResult3).cancel();
        streamTask.finishInput();
        task.waitForTaskCompletion(false);
    }
}

Example 22 with MockEnvironment

use of org.apache.flink.runtime.operators.testutils.MockEnvironment in project flink by apache.

the class StreamTaskTest method testProcessWithUnAvailableOutput.

@Test
public void testProcessWithUnAvailableOutput() throws Exception {
    final long sleepTimeOutsideMail = 42;
    final long sleepTimeInsideMail = 44;
    @Nullable WaitingThread waitingThread = null;
    try (final MockEnvironment environment = setupEnvironment(true, false)) {
        final int numberOfProcessCalls = 10;
        final AvailabilityTestInputProcessor inputProcessor = new AvailabilityTestInputProcessor(numberOfProcessCalls);
        final StreamTask task = new MockStreamTaskBuilder(environment).setStreamInputProcessor(inputProcessor).build();
        final MailboxExecutor executor = task.mailboxProcessor.getMainMailboxExecutor();
        TaskIOMetricGroup ioMetricGroup = task.getEnvironment().getMetricGroup().getIOMetricGroup();
        final RunnableWithException completeFutureTask = () -> {
            assertEquals(1, inputProcessor.currentNumProcessCalls);
            assertFalse(task.mailboxProcessor.isDefaultActionAvailable());
            environment.getWriter(1).getAvailableFuture().complete(null);
        };
        waitingThread = new WaitingThread(executor, completeFutureTask, sleepTimeInsideMail, sleepTimeOutsideMail, ioMetricGroup.getSoftBackPressuredTimePerSecond());
        // Make sure WaitingThread is started after Task starts processing.
        executor.submit(waitingThread::start, "This task will submit another task to execute after processing input once.");
        long startTs = System.currentTimeMillis();
        task.invoke();
        long totalDuration = System.currentTimeMillis() - startTs;
        assertThat(ioMetricGroup.getSoftBackPressuredTimePerSecond().getCount(), greaterThanOrEqualTo(sleepTimeOutsideMail));
        assertThat(ioMetricGroup.getSoftBackPressuredTimePerSecond().getCount(), Matchers.lessThanOrEqualTo(totalDuration - sleepTimeInsideMail));
        assertThat(ioMetricGroup.getIdleTimeMsPerSecond().getCount(), is(0L));
        assertEquals(numberOfProcessCalls, inputProcessor.currentNumProcessCalls);
    } finally {
        if (waitingThread != null) {
            waitingThread.join();
        }
    }
}

Example 23 with MockEnvironment

use of org.apache.flink.runtime.operators.testutils.MockEnvironment in project flink by apache.

the class StreamTaskTest method testThreadInvariants.

/**
 * Tests that some StreamTask methods are called only in the main task's thread. Currently, the
 * main task's thread is the thread that creates the task.
 */
@Test
public void testThreadInvariants() throws Throwable {
    Configuration taskConfiguration = new Configuration();
    StreamConfig streamConfig = new StreamConfig(taskConfiguration);
    streamConfig.setStreamOperator(new StreamMap<>(value -> value));
    streamConfig.setOperatorID(new OperatorID());
    try (MockEnvironment mockEnvironment = new MockEnvironmentBuilder().setTaskConfiguration(taskConfiguration).build()) {
        ClassLoader taskClassLoader = new TestUserCodeClassLoader();
        RunningTask<ThreadInspectingTask> runningTask = runTask(() -> {
            Thread.currentThread().setContextClassLoader(taskClassLoader);
            return new ThreadInspectingTask(mockEnvironment);
        });
        runningTask.invocationFuture.get();
        assertThat(runningTask.streamTask.getTaskClassLoader(), is(sameInstance(taskClassLoader)));
    }
}

Example 24 with MockEnvironment

use of org.apache.flink.runtime.operators.testutils.MockEnvironment in project flink by apache.

the class StreamTaskTest method setupEnvironment.

private MockEnvironment setupEnvironment(boolean... outputAvailabilities) {
    final Configuration configuration = new Configuration();
    new MockStreamConfig(configuration, outputAvailabilities.length);
    final List<ResultPartitionWriter> writers = new ArrayList<>(outputAvailabilities.length);
    for (int i = 0; i < outputAvailabilities.length; i++) {
        writers.add(new AvailabilityTestResultPartitionWriter(outputAvailabilities[i]));
    }
    final MockEnvironment environment = new MockEnvironmentBuilder().setTaskConfiguration(configuration).build();
    environment.addOutputs(writers);
    return environment;
}

Example 25 with MockEnvironment

use of org.apache.flink.runtime.operators.testutils.MockEnvironment in project flink by apache.

the class StreamOperatorWrapperTest method setup.

@Before
public void setup() throws Exception {
    this.operatorWrappers = new ArrayList<>();
    this.output = new ConcurrentLinkedQueue<>();
    try (MockEnvironment env = MockEnvironment.builder().build()) {
        this.containingTask = new MockStreamTaskBuilder(env).build();
        // initialize operator wrappers
        for (int i = 0; i < numOperators; i++) {
            MailboxExecutor mailboxExecutor = containingTask.getMailboxExecutorFactory().createExecutor(i);
            TimerMailController timerMailController = new TimerMailController(containingTask, mailboxExecutor);
            ProcessingTimeServiceImpl processingTimeService = new ProcessingTimeServiceImpl(timerService, timerMailController::wrapCallback);
            TestOneInputStreamOperator streamOperator = new TestOneInputStreamOperator("Operator" + i, output, processingTimeService, mailboxExecutor, timerMailController);
            streamOperator.setProcessingTimeService(processingTimeService);
            StreamOperatorWrapper<?, ?> operatorWrapper = new StreamOperatorWrapper<>(streamOperator, Optional.ofNullable(streamOperator.getProcessingTimeService()), mailboxExecutor, i == 0);
            operatorWrappers.add(operatorWrapper);
        }
        StreamOperatorWrapper<?, ?> previous = null;
        for (StreamOperatorWrapper<?, ?> current : operatorWrappers) {
            if (previous != null) {
                previous.setNext(current);
            }
            current.setPrevious(previous);
            previous = current;
        }
    }
}