Examples with MockEnvironmentBuilder org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder used on opensource projects

Example 16 with MockEnvironmentBuilder

use of org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder in project flink by apache.

the class StreamTaskTest method testEmptySubtaskStateLeadsToStatelessAcknowledgment.

/**
 * FLINK-5985
 *
 * <p>This test ensures that empty snapshots (no op/keyed stated whatsoever) will be reported as
 * stateless tasks. This happens by translating an empty {@link SubtaskState} into reporting
 * 'null' to #acknowledgeCheckpoint.
 */
@Test
public void testEmptySubtaskStateLeadsToStatelessAcknowledgment() throws Exception {
    // latch blocks until the async checkpoint thread acknowledges
    final OneShotLatch checkpointCompletedLatch = new OneShotLatch();
    final List<SubtaskState> checkpointResult = new ArrayList<>(1);
    CheckpointResponder checkpointResponder = mock(CheckpointResponder.class);
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            SubtaskState subtaskState = invocation.getArgument(4);
            checkpointResult.add(subtaskState);
            checkpointCompletedLatch.trigger();
            return null;
        }
    }).when(checkpointResponder).acknowledgeCheckpoint(any(JobID.class), any(ExecutionAttemptID.class), anyLong(), any(CheckpointMetrics.class), nullable(TaskStateSnapshot.class));
    TaskStateManager taskStateManager = new TaskStateManagerImpl(new JobID(1L, 2L), new ExecutionAttemptID(), mock(TaskLocalStateStoreImpl.class), new InMemoryStateChangelogStorage(), null, checkpointResponder);
    // mock the operator with empty snapshot result (all state handles are null)
    OneInputStreamOperator<String, String> statelessOperator = streamOperatorWithSnapshot(new OperatorSnapshotFutures());
    try (MockEnvironment mockEnvironment = new MockEnvironmentBuilder().setTaskStateManager(taskStateManager).build()) {
        RunningTask<MockStreamTask> task = runTask(() -> createMockStreamTask(mockEnvironment, operatorChain(statelessOperator)));
        waitTaskIsRunning(task.streamTask, task.invocationFuture);
        task.streamTask.triggerCheckpointAsync(new CheckpointMetaData(42L, 1L), CheckpointOptions.forCheckpointWithDefaultLocation());
        checkpointCompletedLatch.await(30, TimeUnit.SECONDS);
        // ensure that 'null' was acknowledged as subtask state
        Assert.assertNull(checkpointResult.get(0));
        task.streamTask.cancel();
        task.waitForTaskCompletion(true);
    }
}

Example 17 with MockEnvironmentBuilder

use of org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder in project flink by apache.

the class SourceFunctionUtil method runRichSourceFunction.

private static <T extends Serializable> List<T> runRichSourceFunction(SourceFunction<T> sourceFunction) throws Exception {
    try (MockEnvironment environment = new MockEnvironmentBuilder().setTaskName("MockTask").setManagedMemorySize(3 * 1024 * 1024).setInputSplitProvider(new MockInputSplitProvider()).setBufferSize(1024).build()) {
        AbstractStreamOperator<?> operator = mock(AbstractStreamOperator.class);
        when(operator.getExecutionConfig()).thenReturn(new ExecutionConfig());
        RuntimeContext runtimeContext = new StreamingRuntimeContext(operator, environment, new HashMap<>());
        ((RichFunction) sourceFunction).setRuntimeContext(runtimeContext);
        ((RichFunction) sourceFunction).open(new Configuration());
        return runNonRichSourceFunction(sourceFunction);
    }
}

Example 18 with MockEnvironmentBuilder

use of org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder in project flink by apache.

the class StreamTaskSystemExitTest method testCancelSystemExitStreamTask.

@Test(expected = UserSystemExitException.class)
public void testCancelSystemExitStreamTask() throws Exception {
    Environment mockEnvironment = new MockEnvironmentBuilder().build();
    SystemExitStreamTask systemExitStreamTask = new SystemExitStreamTask(mockEnvironment, SystemExitStreamTask.ExitPoint.CANCEL);
    systemExitStreamTask.cancel();
}

Example 19 with MockEnvironmentBuilder

use of org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder in project flink by apache.

the class RocksDBStateBackendConfigTest method testConfigureTimerServiceLoadingFromApplication.

/**
 * Validates that user custom configuration from code should override the flink-conf.yaml.
 */
@Test
public void testConfigureTimerServiceLoadingFromApplication() throws Exception {
    final MockEnvironment env = new MockEnvironmentBuilder().build();
    // priorityQueueStateType of the job backend
    final EmbeddedRocksDBStateBackend backend = new EmbeddedRocksDBStateBackend();
    backend.setPriorityQueueStateType(EmbeddedRocksDBStateBackend.PriorityQueueStateType.HEAP);
    // priorityQueueStateType in the cluster config
    final Configuration configFromConfFile = new Configuration();
    configFromConfFile.setString(RocksDBOptions.TIMER_SERVICE_FACTORY.key(), RocksDBStateBackend.PriorityQueueStateType.ROCKSDB.toString());
    // configure final backend from job and cluster config
    final EmbeddedRocksDBStateBackend configuredRocksDBStateBackend = backend.configure(configFromConfFile, Thread.currentThread().getContextClassLoader());
    final RocksDBKeyedStateBackend<Integer> keyedBackend = createKeyedStateBackend(configuredRocksDBStateBackend, env, IntSerializer.INSTANCE);
    // priorityQueueStateType of the job backend should be preserved
    assertThat(keyedBackend.getPriorityQueueFactory(), instanceOf(HeapPriorityQueueSetFactory.class));
    keyedBackend.close();
    keyedBackend.dispose();
    env.close();
}

Example 20 with MockEnvironmentBuilder

use of org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder in project flink by apache.

the class ChainedOperatorsMetricTest method testOperatorIOMetricReuse.

@Test
public void testOperatorIOMetricReuse() throws Exception {
    // environment
    initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
    this.mockEnv = new MockEnvironmentBuilder().setTaskName(HEAD_OPERATOR_NAME).setManagedMemorySize(MEMORY_MANAGER_SIZE).setInputSplitProvider(this.inputSplitProvider).setBufferSize(NETWORK_BUFFER_SIZE).setMetricGroup(TaskManagerMetricGroup.createTaskManagerMetricGroup(NoOpMetricRegistry.INSTANCE, "host", ResourceID.generate()).addJob(new JobID(), "jobName").addTask(new JobVertexID(), new ExecutionAttemptID(), "task", 0, 0)).build();
    final int keyCnt = 100;
    final int valCnt = 20;
    final int numRecords = keyCnt * valCnt;
    addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
    addOutput(this.outList);
    // the chained operator
    addChainedOperator();
    // creates the head operator and assembles the chain
    registerTask(FlatMapDriver.class, DuplicatingFlatMapFunction.class);
    final BatchTask<FlatMapFunction<Record, Record>, Record> testTask = new BatchTask<>(this.mockEnv);
    testTask.invoke();
    Assert.assertEquals(numRecords * 2 * 2, this.outList.size());
    final TaskMetricGroup taskMetricGroup = mockEnv.getMetricGroup();
    // verify task-level metrics
    {
        final TaskIOMetricGroup ioMetricGroup = taskMetricGroup.getIOMetricGroup();
        final Counter numRecordsInCounter = ioMetricGroup.getNumRecordsInCounter();
        final Counter numRecordsOutCounter = ioMetricGroup.getNumRecordsOutCounter();
        Assert.assertEquals(numRecords, numRecordsInCounter.getCount());
        Assert.assertEquals(numRecords * 2 * 2, numRecordsOutCounter.getCount());
    }
    // verify head operator metrics
    {
        // this only returns the existing group and doesn't create a new one
        final OperatorMetricGroup operatorMetricGroup1 = taskMetricGroup.getOrAddOperator(HEAD_OPERATOR_NAME);
        final OperatorIOMetricGroup ioMetricGroup = operatorMetricGroup1.getIOMetricGroup();
        final Counter numRecordsInCounter = ioMetricGroup.getNumRecordsInCounter();
        final Counter numRecordsOutCounter = ioMetricGroup.getNumRecordsOutCounter();
        Assert.assertEquals(numRecords, numRecordsInCounter.getCount());
        Assert.assertEquals(numRecords * 2, numRecordsOutCounter.getCount());
    }
    // verify chained operator metrics
    {
        // this only returns the existing group and doesn't create a new one
        final InternalOperatorMetricGroup operatorMetricGroup1 = taskMetricGroup.getOrAddOperator(CHAINED_OPERATOR_NAME);
        final InternalOperatorIOMetricGroup ioMetricGroup = operatorMetricGroup1.getIOMetricGroup();
        final Counter numRecordsInCounter = ioMetricGroup.getNumRecordsInCounter();
        final Counter numRecordsOutCounter = ioMetricGroup.getNumRecordsOutCounter();
        Assert.assertEquals(numRecords * 2, numRecordsInCounter.getCount());
        Assert.assertEquals(numRecords * 2 * 2, numRecordsOutCounter.getCount());
    }
}