Examples with MockStreamTaskBuilder org.apache.flink.streaming.util.MockStreamTaskBuilder used on opensource projects

Example 1 with MockStreamTaskBuilder

use of org.apache.flink.streaming.util.MockStreamTaskBuilder in project flink by apache.

the class OperatorChainTest method setupOperatorChain.

// ------------------------------------------------------------------------
// Operator Chain Setup Utils
// ------------------------------------------------------------------------
@SafeVarargs
public static <T, OP extends StreamOperator<T>> OperatorChain<T, OP> setupOperatorChain(OneInputStreamOperator<T, T>... operators) throws Exception {
    checkNotNull(operators);
    checkArgument(operators.length > 0);
    try (MockEnvironment env = MockEnvironment.builder().build()) {
        final StreamTask<?, ?> containingTask = new MockStreamTaskBuilder(env).build();
        final StreamConfig cfg = new StreamConfig(new Configuration());
        cfg.setOperatorID(new OperatorID());
        cfg.setStateKeySerializer(new StringSerializer());
        final List<StreamOperatorWrapper<?, ?>> operatorWrappers = new ArrayList<>();
        // initial output goes to nowhere
        @SuppressWarnings({ "unchecked", "rawtypes" }) WatermarkGaugeExposingOutput<StreamRecord<T>> lastWriter = new BroadcastingOutputCollector<>(new Output[0]);
        // build the reverse operators array
        for (int i = 0; i < operators.length; i++) {
            int operatorIndex = operators.length - i - 1;
            OneInputStreamOperator<T, T> op = operators[operatorIndex];
            if (op instanceof SetupableStreamOperator) {
                ((SetupableStreamOperator) op).setup(containingTask, cfg, lastWriter);
            }
            lastWriter = new ChainingOutput<>(op, null);
            ProcessingTimeService processingTimeService = null;
            if (op instanceof AbstractStreamOperator) {
                processingTimeService = ((AbstractStreamOperator) op).getProcessingTimeService();
            }
            operatorWrappers.add(new StreamOperatorWrapper<>(op, Optional.ofNullable(processingTimeService), containingTask.getMailboxExecutorFactory().createExecutor(i), operatorIndex == 0));
        }
        @SuppressWarnings("unchecked") final StreamOperatorWrapper<T, OP> headOperatorWrapper = (StreamOperatorWrapper<T, OP>) operatorWrappers.get(operatorWrappers.size() - 1);
        return new RegularOperatorChain<>(operatorWrappers, new RecordWriterOutput<?>[0], lastWriter, headOperatorWrapper);
    }
}

Example 2 with MockStreamTaskBuilder

use of org.apache.flink.streaming.util.MockStreamTaskBuilder in project flink by apache.

the class InputProcessorUtilTest method testCreateCheckpointedMultipleInputGate.

@Test
public void testCreateCheckpointedMultipleInputGate() throws Exception {
    try (CloseableRegistry registry = new CloseableRegistry()) {
        MockEnvironment environment = new MockEnvironmentBuilder().build();
        MockStreamTask streamTask = new MockStreamTaskBuilder(environment).build();
        StreamConfig streamConfig = new StreamConfig(environment.getJobConfiguration());
        streamConfig.setCheckpointMode(CheckpointingMode.EXACTLY_ONCE);
        streamConfig.setUnalignedCheckpointsEnabled(true);
        // First input gate has index larger than the second
        List<IndexedInputGate>[] inputGates = new List[] { Collections.singletonList(getGate(1, 4)), Collections.singletonList(getGate(0, 2)) };
        CheckpointBarrierHandler barrierHandler = InputProcessorUtil.createCheckpointBarrierHandler(streamTask, streamConfig, new TestSubtaskCheckpointCoordinator(new MockChannelStateWriter()), streamTask.getName(), inputGates, Collections.emptyList(), new SyncMailboxExecutor(), new TestProcessingTimeService());
        CheckpointedInputGate[] checkpointedMultipleInputGate = InputProcessorUtil.createCheckpointedMultipleInputGate(new SyncMailboxExecutor(), inputGates, environment.getMetricGroup().getIOMetricGroup(), barrierHandler, streamConfig);
        for (CheckpointedInputGate checkpointedInputGate : checkpointedMultipleInputGate) {
            registry.registerCloseable(checkpointedInputGate);
        }
        List<IndexedInputGate> allInputGates = Arrays.stream(inputGates).flatMap(gates -> gates.stream()).collect(Collectors.toList());
        for (IndexedInputGate inputGate : allInputGates) {
            for (int channelId = 0; channelId < inputGate.getNumberOfInputChannels(); channelId++) {
                barrierHandler.processBarrier(new CheckpointBarrier(1, 42, CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, CheckpointStorageLocationReference.getDefault())), new InputChannelInfo(inputGate.getGateIndex(), channelId), false);
            }
        }
        assertTrue(barrierHandler.getAllBarriersReceivedFuture(1).isDone());
    }
}

Example 3 with MockStreamTaskBuilder

use of org.apache.flink.streaming.util.MockStreamTaskBuilder in project flink by apache.

the class StreamTaskTest method testProcessWithAvailableOutput.

@Test
public void testProcessWithAvailableOutput() throws Exception {
    try (final MockEnvironment environment = setupEnvironment(true, true)) {
        final int numberOfProcessCalls = 10;
        final AvailabilityTestInputProcessor inputProcessor = new AvailabilityTestInputProcessor(numberOfProcessCalls);
        final StreamTask task = new MockStreamTaskBuilder(environment).setStreamInputProcessor(inputProcessor).build();
        task.invoke();
        assertEquals(numberOfProcessCalls, inputProcessor.currentNumProcessCalls);
    }
}

Example 4 with MockStreamTaskBuilder

use of org.apache.flink.streaming.util.MockStreamTaskBuilder in project flink by apache.

the class StreamTaskTest method testQuiesceOfMailboxRightBeforeSubmittingActionViaTimerService.

@Test
public void testQuiesceOfMailboxRightBeforeSubmittingActionViaTimerService() throws Exception {
    // given: the stream task with configured handle async exception.
    AtomicBoolean submitThroughputFail = new AtomicBoolean();
    MockEnvironment mockEnvironment = new MockEnvironmentBuilder().build();
    final UnAvailableTestInputProcessor inputProcessor = new UnAvailableTestInputProcessor();
    RunningTask<StreamTask<?, ?>> task = runTask(() -> new MockStreamTaskBuilder(mockEnvironment).setHandleAsyncException((str, t) -> submitThroughputFail.set(true)).setStreamInputProcessor(inputProcessor).build());
    waitTaskIsRunning(task.streamTask, task.invocationFuture);
    TimerService timerService = task.streamTask.systemTimerService;
    MailboxExecutor mainMailboxExecutor = task.streamTask.mailboxProcessor.getMainMailboxExecutor();
    CountDownLatch stoppingMailboxLatch = new CountDownLatch(1);
    timerService.registerTimer(timerService.getCurrentProcessingTime(), (time) -> {
        stoppingMailboxLatch.await();
        // The time to the start 'afterInvoke' inside of mailbox.
        // 'afterInvoke' won't finish until this execution won't finish so it is
        // impossible to wait on latch or something else.
        Thread.sleep(5);
        mainMailboxExecutor.submit(() -> {
        }, "test");
    });
    // when: Calling the quiesce for mailbox and finishing the timer service.
    mainMailboxExecutor.submit(() -> {
        stoppingMailboxLatch.countDown();
        task.streamTask.afterInvoke();
    }, "test").get();
    // then: the exception handle wasn't invoked because the such situation is expected.
    assertFalse(submitThroughputFail.get());
    // Correctly shutdown the stream task to avoid hanging.
    inputProcessor.availabilityProvider.getUnavailableToResetAvailable().complete(null);
}

Example 5 with MockStreamTaskBuilder

use of org.apache.flink.streaming.util.MockStreamTaskBuilder in project flink by apache.

the class StreamTaskTest method testProcessWithUnAvailableOutput.

@Test
public void testProcessWithUnAvailableOutput() throws Exception {
    final long sleepTimeOutsideMail = 42;
    final long sleepTimeInsideMail = 44;
    @Nullable WaitingThread waitingThread = null;
    try (final MockEnvironment environment = setupEnvironment(true, false)) {
        final int numberOfProcessCalls = 10;
        final AvailabilityTestInputProcessor inputProcessor = new AvailabilityTestInputProcessor(numberOfProcessCalls);
        final StreamTask task = new MockStreamTaskBuilder(environment).setStreamInputProcessor(inputProcessor).build();
        final MailboxExecutor executor = task.mailboxProcessor.getMainMailboxExecutor();
        TaskIOMetricGroup ioMetricGroup = task.getEnvironment().getMetricGroup().getIOMetricGroup();
        final RunnableWithException completeFutureTask = () -> {
            assertEquals(1, inputProcessor.currentNumProcessCalls);
            assertFalse(task.mailboxProcessor.isDefaultActionAvailable());
            environment.getWriter(1).getAvailableFuture().complete(null);
        };
        waitingThread = new WaitingThread(executor, completeFutureTask, sleepTimeInsideMail, sleepTimeOutsideMail, ioMetricGroup.getSoftBackPressuredTimePerSecond());
        // Make sure WaitingThread is started after Task starts processing.
        executor.submit(waitingThread::start, "This task will submit another task to execute after processing input once.");
        long startTs = System.currentTimeMillis();
        task.invoke();
        long totalDuration = System.currentTimeMillis() - startTs;
        assertThat(ioMetricGroup.getSoftBackPressuredTimePerSecond().getCount(), greaterThanOrEqualTo(sleepTimeOutsideMail));
        assertThat(ioMetricGroup.getSoftBackPressuredTimePerSecond().getCount(), Matchers.lessThanOrEqualTo(totalDuration - sleepTimeInsideMail));
        assertThat(ioMetricGroup.getIdleTimeMsPerSecond().getCount(), is(0L));
        assertEquals(numberOfProcessCalls, inputProcessor.currentNumProcessCalls);
    } finally {
        if (waitingThread != null) {
            waitingThread.join();
        }
    }
}