Examples with MockStreamTaskBuilder org.apache.flink.streaming.util.MockStreamTaskBuilder used on opensource projects

Example 6 with MockStreamTaskBuilder

use of org.apache.flink.streaming.util.MockStreamTaskBuilder in project flink by apache.

the class StreamOperatorWrapperTest method setup.

@Before
public void setup() throws Exception {
    this.operatorWrappers = new ArrayList<>();
    this.output = new ConcurrentLinkedQueue<>();
    try (MockEnvironment env = MockEnvironment.builder().build()) {
        this.containingTask = new MockStreamTaskBuilder(env).build();
        // initialize operator wrappers
        for (int i = 0; i < numOperators; i++) {
            MailboxExecutor mailboxExecutor = containingTask.getMailboxExecutorFactory().createExecutor(i);
            TimerMailController timerMailController = new TimerMailController(containingTask, mailboxExecutor);
            ProcessingTimeServiceImpl processingTimeService = new ProcessingTimeServiceImpl(timerService, timerMailController::wrapCallback);
            TestOneInputStreamOperator streamOperator = new TestOneInputStreamOperator("Operator" + i, output, processingTimeService, mailboxExecutor, timerMailController);
            streamOperator.setProcessingTimeService(processingTimeService);
            StreamOperatorWrapper<?, ?> operatorWrapper = new StreamOperatorWrapper<>(streamOperator, Optional.ofNullable(streamOperator.getProcessingTimeService()), mailboxExecutor, i == 0);
            operatorWrappers.add(operatorWrapper);
        }
        StreamOperatorWrapper<?, ?> previous = null;
        for (StreamOperatorWrapper<?, ?> current : operatorWrappers) {
            if (previous != null) {
                previous.setNext(current);
            }
            current.setPrevious(previous);
            previous = current;
        }
    }
}

Example 7 with MockStreamTaskBuilder

use of org.apache.flink.streaming.util.MockStreamTaskBuilder in project flink by apache.

the class SubtaskCheckpointCoordinatorTest method testSavepointNotResultingInPriorityEvents.

@Test
public void testSavepointNotResultingInPriorityEvents() throws Exception {
    MockEnvironment mockEnvironment = MockEnvironment.builder().build();
    try (SubtaskCheckpointCoordinator coordinator = new MockSubtaskCheckpointCoordinatorBuilder().setUnalignedCheckpointEnabled(true).setEnvironment(mockEnvironment).build()) {
        AtomicReference<Boolean> broadcastedPriorityEvent = new AtomicReference<>(null);
        final OperatorChain<?, ?> operatorChain = new RegularOperatorChain(new MockStreamTaskBuilder(mockEnvironment).build(), new NonRecordWriter<>()) {

            @Override
            public void broadcastEvent(AbstractEvent event, boolean isPriorityEvent) throws IOException {
                super.broadcastEvent(event, isPriorityEvent);
                broadcastedPriorityEvent.set(isPriorityEvent);
            }
        };
        coordinator.checkpointState(new CheckpointMetaData(0, 0), new CheckpointOptions(SavepointType.savepoint(SavepointFormatType.CANONICAL), CheckpointStorageLocationReference.getDefault()), new CheckpointMetricsBuilder(), operatorChain, false, () -> true);
        assertEquals(false, broadcastedPriorityEvent.get());
    }
}

Example 8 with MockStreamTaskBuilder

use of org.apache.flink.streaming.util.MockStreamTaskBuilder in project flink by apache.

the class SubtaskCheckpointCoordinatorTest method testForceAlignedCheckpointResultingInPriorityEvents.

@Test
public void testForceAlignedCheckpointResultingInPriorityEvents() throws Exception {
    final long checkpointId = 42L;
    MockEnvironment mockEnvironment = MockEnvironment.builder().build();
    try (SubtaskCheckpointCoordinator coordinator = new MockSubtaskCheckpointCoordinatorBuilder().setUnalignedCheckpointEnabled(true).setEnvironment(mockEnvironment).build()) {
        AtomicReference<Boolean> broadcastedPriorityEvent = new AtomicReference<>(null);
        final OperatorChain<?, ?> operatorChain = new RegularOperatorChain(new MockStreamTaskBuilder(mockEnvironment).build(), new NonRecordWriter<>()) {

            @Override
            public void broadcastEvent(AbstractEvent event, boolean isPriorityEvent) throws IOException {
                super.broadcastEvent(event, isPriorityEvent);
                broadcastedPriorityEvent.set(isPriorityEvent);
                // test if we can write output data
                coordinator.getChannelStateWriter().addOutputData(checkpointId, new ResultSubpartitionInfo(0, 0), 0, BufferBuilderTestUtils.buildSomeBuffer(500));
            }
        };
        CheckpointOptions forcedAlignedOptions = CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, CheckpointStorageLocationReference.getDefault()).withUnalignedUnsupported();
        coordinator.checkpointState(new CheckpointMetaData(checkpointId, 0), forcedAlignedOptions, new CheckpointMetricsBuilder(), operatorChain, false, () -> true);
        assertEquals(true, broadcastedPriorityEvent.get());
    }
}

Example 9 with MockStreamTaskBuilder

use of org.apache.flink.streaming.util.MockStreamTaskBuilder in project flink by apache.

the class StreamTaskTest method testProcessWithUnAvailableInput.

@Test
public void testProcessWithUnAvailableInput() throws Exception {
    final long sleepTimeOutsideMail = 42;
    final long sleepTimeInsideMail = 44;
    @Nullable WaitingThread waitingThread = null;
    try (final MockEnvironment environment = setupEnvironment(true, true)) {
        final UnAvailableTestInputProcessor inputProcessor = new UnAvailableTestInputProcessor();
        final StreamTask task = new MockStreamTaskBuilder(environment).setStreamInputProcessor(inputProcessor).build();
        TaskIOMetricGroup ioMetricGroup = task.getEnvironment().getMetricGroup().getIOMetricGroup();
        final MailboxExecutor executor = task.mailboxProcessor.getMainMailboxExecutor();
        final RunnableWithException completeFutureTask = () -> {
            inputProcessor.availabilityProvider.getUnavailableToResetAvailable().complete(null);
        };
        waitingThread = new WaitingThread(executor, completeFutureTask, sleepTimeInsideMail, sleepTimeOutsideMail, ioMetricGroup.getIdleTimeMsPerSecond());
        // Make sure WaitingThread is started after Task starts processing.
        executor.submit(waitingThread::start, "Start WaitingThread after Task starts processing input.");
        SystemClock clock = SystemClock.getInstance();
        long startTs = clock.absoluteTimeMillis();
        task.invoke();
        long totalDuration = clock.absoluteTimeMillis() - startTs;
        assertThat(ioMetricGroup.getIdleTimeMsPerSecond().getCount(), greaterThanOrEqualTo(sleepTimeOutsideMail));
        assertThat(ioMetricGroup.getIdleTimeMsPerSecond().getCount(), Matchers.lessThanOrEqualTo(totalDuration - sleepTimeInsideMail));
        assertThat(ioMetricGroup.getSoftBackPressuredTimePerSecond().getCount(), is(0L));
        assertThat(ioMetricGroup.getHardBackPressuredTimePerSecond().getCount(), is(0L));
    } finally {
        if (waitingThread != null) {
            waitingThread.join();
        }
    }
}

Example 10 with MockStreamTaskBuilder

use of org.apache.flink.streaming.util.MockStreamTaskBuilder in project flink by apache.

the class StreamSourceOperatorWatermarksTest method setupSourceOperator.

// ------------------------------------------------------------------------
@SuppressWarnings("unchecked")
private static <T> MockStreamTask setupSourceOperator(StreamSource<T, ?> operator, TimeCharacteristic timeChar, long watermarkInterval, final TimerService timeProvider) throws Exception {
    ExecutionConfig executionConfig = new ExecutionConfig();
    executionConfig.setAutoWatermarkInterval(watermarkInterval);
    StreamConfig cfg = new StreamConfig(new Configuration());
    cfg.setStateBackend(new MemoryStateBackend());
    cfg.setTimeCharacteristic(timeChar);
    cfg.setOperatorID(new OperatorID());
    Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);
    MockStreamTask mockTask = new MockStreamTaskBuilder(env).setConfig(cfg).setExecutionConfig(executionConfig).setTimerService(timeProvider).build();
    operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
    return mockTask;
}