Examples with MockEnvironmentBuilder org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder used on opensource projects

Example 6 with MockEnvironmentBuilder

use of org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder in project flink by apache.

the class InputFormatSourceFunctionTest method testFormatLifecycle.

private void testFormatLifecycle(final boolean midCancel) throws Exception {
    final int noOfSplits = 5;
    final int cancelAt = 2;
    final LifeCycleTestInputFormat format = new LifeCycleTestInputFormat();
    final InputFormatSourceFunction<Integer> reader = new InputFormatSourceFunction<>(format, TypeInformation.of(Integer.class));
    try (MockEnvironment environment = new MockEnvironmentBuilder().setTaskName("no").setManagedMemorySize(4 * MemoryManager.DEFAULT_PAGE_SIZE).build()) {
        reader.setRuntimeContext(new MockRuntimeContext(format, noOfSplits, environment));
        Assert.assertTrue(!format.isConfigured);
        Assert.assertTrue(!format.isInputFormatOpen);
        Assert.assertTrue(!format.isSplitOpen);
        reader.open(new Configuration());
        Assert.assertTrue(format.isConfigured);
        TestSourceContext ctx = new TestSourceContext(reader, format, midCancel, cancelAt);
        reader.run(ctx);
        int splitsSeen = ctx.getSplitsSeen();
        Assert.assertTrue(midCancel ? splitsSeen == cancelAt : splitsSeen == noOfSplits);
        // we have exhausted the splits so the
        // format and splits should be closed by now
        Assert.assertTrue(!format.isSplitOpen);
        Assert.assertTrue(!format.isInputFormatOpen);
    }
}

Example 7 with MockEnvironmentBuilder

use of org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder in project flink by apache.

the class SourceOperatorLatencyMetricsTest method testLatencyMarkEmission.

private void testLatencyMarkEmission(boolean shouldExpectLatencyMarkers, Configuration taskManagerConfig, ExecutionConfig executionConfig) throws Exception {
    try (SourceOperatorTestHarness testHarness = new SourceOperatorTestHarness(new SourceOperatorFactory(new MockSource(Boundedness.CONTINUOUS_UNBOUNDED, 1), WatermarkStrategy.noWatermarks()), new MockEnvironmentBuilder().setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(taskManagerConfig)).setExecutionConfig(executionConfig).build())) {
        testHarness.open();
        testHarness.setup();
        for (long processingTime = 0; processingTime <= MAX_PROCESSING_TIME; processingTime++) {
            testHarness.getProcessingTimeService().setCurrentTime(processingTime);
            testHarness.emitNext();
        }
        List<LatencyMarker> expectedOutput = new ArrayList<>();
        if (!shouldExpectLatencyMarkers) {
            assertTrue(testHarness.getOutput().isEmpty());
        } else {
            expectedOutput.add(new LatencyMarker(1, testHarness.getOperator().getOperatorID(), 0));
            for (long markedTime = LATENCY_MARK_INTERVAL; markedTime <= MAX_PROCESSING_TIME; markedTime += LATENCY_MARK_INTERVAL) {
                expectedOutput.add(new LatencyMarker(markedTime, testHarness.getOperator().getOperatorID(), 0));
            }
            assertThat((Collection<Object>) testHarness.getOutput(), contains(expectedOutput.toArray()));
        }
    }
}

Example 8 with MockEnvironmentBuilder

use of org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder in project flink by apache.

the class StreamSourceOperatorLatencyMetricsTest method testLatencyMarkEmission.

private void testLatencyMarkEmission(int numberLatencyMarkers, OperatorSetupOperation operatorSetup) throws Exception {
    final List<StreamElement> output = new ArrayList<>();
    final TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService();
    testProcessingTimeService.setCurrentTime(0L);
    final List<Long> processingTimes = Arrays.asList(1L, 10L, 11L, 21L, maxProcessingTime);
    // regular stream source operator
    final StreamSource<Long, ProcessingTimeServiceSource> operator = new StreamSource<>(new ProcessingTimeServiceSource(testProcessingTimeService, processingTimes));
    operatorSetup.setupSourceOperator(operator, testProcessingTimeService);
    // run and wait to be stopped
    OperatorChain<?, ?> operatorChain = new RegularOperatorChain<>(operator.getContainingTask(), StreamTask.createRecordWriterDelegate(operator.getOperatorConfig(), new MockEnvironmentBuilder().build()));
    try {
        operator.run(new Object(), new CollectorOutput<>(output), operatorChain);
        operator.finish();
    } finally {
        operatorChain.close();
    }
    assertEquals(numberLatencyMarkers, output.size());
    long timestamp = 0L;
    int expectedLatencyIndex = 0;
    int i = 0;
    // verify that its only latency markers
    for (; i < numberLatencyMarkers; i++) {
        StreamElement se = output.get(i);
        Assert.assertTrue(se.isLatencyMarker());
        Assert.assertEquals(operator.getOperatorID(), se.asLatencyMarker().getOperatorId());
        Assert.assertEquals(0, se.asLatencyMarker().getSubtaskIndex());
        // as a result of which we never emit both 10 and 11
        while (timestamp > processingTimes.get(expectedLatencyIndex)) {
            expectedLatencyIndex++;
        }
        Assert.assertEquals(processingTimes.get(expectedLatencyIndex).longValue(), se.asLatencyMarker().getMarkedTime());
        timestamp += latencyMarkInterval;
    }
}

Example 9 with MockEnvironmentBuilder

use of org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder in project flink by apache.

the class InputProcessorUtilTest method testCreateCheckpointedMultipleInputGate.

@Test
public void testCreateCheckpointedMultipleInputGate() throws Exception {
    try (CloseableRegistry registry = new CloseableRegistry()) {
        MockEnvironment environment = new MockEnvironmentBuilder().build();
        MockStreamTask streamTask = new MockStreamTaskBuilder(environment).build();
        StreamConfig streamConfig = new StreamConfig(environment.getJobConfiguration());
        streamConfig.setCheckpointMode(CheckpointingMode.EXACTLY_ONCE);
        streamConfig.setUnalignedCheckpointsEnabled(true);
        // First input gate has index larger than the second
        List<IndexedInputGate>[] inputGates = new List[] { Collections.singletonList(getGate(1, 4)), Collections.singletonList(getGate(0, 2)) };
        CheckpointBarrierHandler barrierHandler = InputProcessorUtil.createCheckpointBarrierHandler(streamTask, streamConfig, new TestSubtaskCheckpointCoordinator(new MockChannelStateWriter()), streamTask.getName(), inputGates, Collections.emptyList(), new SyncMailboxExecutor(), new TestProcessingTimeService());
        CheckpointedInputGate[] checkpointedMultipleInputGate = InputProcessorUtil.createCheckpointedMultipleInputGate(new SyncMailboxExecutor(), inputGates, environment.getMetricGroup().getIOMetricGroup(), barrierHandler, streamConfig);
        for (CheckpointedInputGate checkpointedInputGate : checkpointedMultipleInputGate) {
            registry.registerCloseable(checkpointedInputGate);
        }
        List<IndexedInputGate> allInputGates = Arrays.stream(inputGates).flatMap(gates -> gates.stream()).collect(Collectors.toList());
        for (IndexedInputGate inputGate : allInputGates) {
            for (int channelId = 0; channelId < inputGate.getNumberOfInputChannels(); channelId++) {
                barrierHandler.processBarrier(new CheckpointBarrier(1, 42, CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, CheckpointStorageLocationReference.getDefault())), new InputChannelInfo(inputGate.getGateIndex(), channelId), false);
            }
        }
        assertTrue(barrierHandler.getAllBarriersReceivedFuture(1).isDone());
    }
}

Example 10 with MockEnvironmentBuilder

use of org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder in project flink by apache.

the class StreamTaskTest method testQuiesceOfMailboxRightBeforeSubmittingActionViaTimerService.

@Test
public void testQuiesceOfMailboxRightBeforeSubmittingActionViaTimerService() throws Exception {
    // given: the stream task with configured handle async exception.
    AtomicBoolean submitThroughputFail = new AtomicBoolean();
    MockEnvironment mockEnvironment = new MockEnvironmentBuilder().build();
    final UnAvailableTestInputProcessor inputProcessor = new UnAvailableTestInputProcessor();
    RunningTask<StreamTask<?, ?>> task = runTask(() -> new MockStreamTaskBuilder(mockEnvironment).setHandleAsyncException((str, t) -> submitThroughputFail.set(true)).setStreamInputProcessor(inputProcessor).build());
    waitTaskIsRunning(task.streamTask, task.invocationFuture);
    TimerService timerService = task.streamTask.systemTimerService;
    MailboxExecutor mainMailboxExecutor = task.streamTask.mailboxProcessor.getMainMailboxExecutor();
    CountDownLatch stoppingMailboxLatch = new CountDownLatch(1);
    timerService.registerTimer(timerService.getCurrentProcessingTime(), (time) -> {
        stoppingMailboxLatch.await();
        // The time to the start 'afterInvoke' inside of mailbox.
        // 'afterInvoke' won't finish until this execution won't finish so it is
        // impossible to wait on latch or something else.
        Thread.sleep(5);
        mainMailboxExecutor.submit(() -> {
        }, "test");
    });
    // when: Calling the quiesce for mailbox and finishing the timer service.
    mainMailboxExecutor.submit(() -> {
        stoppingMailboxLatch.countDown();
        task.streamTask.afterInvoke();
    }, "test").get();
    // then: the exception handle wasn't invoked because the such situation is expected.
    assertFalse(submitThroughputFail.get());
    // Correctly shutdown the stream task to avoid hanging.
    inputProcessor.availabilityProvider.getUnavailableToResetAvailable().complete(null);
}