Search in sources :

Example 1 with Environment

use of org.apache.flink.runtime.execution.Environment in project flink by apache.

the class RocksDBStateBackendTest method setupRocksKeyedStateBackend.

public void setupRocksKeyedStateBackend() throws Exception {
    blocker = new OneShotLatch();
    waiter = new OneShotLatch();
    testStreamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
    testStreamFactory.setBlockerLatch(blocker);
    testStreamFactory.setWaiterLatch(waiter);
    testStreamFactory.setAfterNumberInvocations(100);
    RocksDBStateBackend backend = getStateBackend();
    Environment env = new DummyEnvironment("TestTask", 1, 0);
    keyedStateBackend = (RocksDBKeyedStateBackend<Integer>) backend.createKeyedStateBackend(env, new JobID(), "Test", IntSerializer.INSTANCE, 2, new KeyGroupRange(0, 1), mock(TaskKvStateRegistry.class));
    testState1 = keyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>("TestState-1", Integer.class, 0));
    testState2 = keyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>("TestState-2", String.class, ""));
    allCreatedCloseables = new ArrayList<>();
    keyedStateBackend.db = spy(keyedStateBackend.db);
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            RocksIterator rocksIterator = spy((RocksIterator) invocationOnMock.callRealMethod());
            allCreatedCloseables.add(rocksIterator);
            return rocksIterator;
        }
    }).when(keyedStateBackend.db).newIterator(any(ColumnFamilyHandle.class), any(ReadOptions.class));
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            Snapshot snapshot = spy((Snapshot) invocationOnMock.callRealMethod());
            allCreatedCloseables.add(snapshot);
            return snapshot;
        }
    }).when(keyedStateBackend.db).getSnapshot();
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ColumnFamilyHandle snapshot = spy((ColumnFamilyHandle) invocationOnMock.callRealMethod());
            allCreatedCloseables.add(snapshot);
            return snapshot;
        }
    }).when(keyedStateBackend.db).createColumnFamily(any(ColumnFamilyDescriptor.class));
    for (int i = 0; i < 100; ++i) {
        keyedStateBackend.setCurrentKey(i);
        testState1.update(4200 + i);
        testState2.update("S-" + (4200 + i));
    }
}
Also used : KeyGroupRange(org.apache.flink.runtime.state.KeyGroupRange) TaskKvStateRegistry(org.apache.flink.runtime.query.TaskKvStateRegistry) DummyEnvironment(org.apache.flink.runtime.operators.testutils.DummyEnvironment) RocksIterator(org.rocksdb.RocksIterator) ColumnFamilyDescriptor(org.rocksdb.ColumnFamilyDescriptor) ColumnFamilyHandle(org.rocksdb.ColumnFamilyHandle) ValueStateDescriptor(org.apache.flink.api.common.state.ValueStateDescriptor) Snapshot(org.rocksdb.Snapshot) ReadOptions(org.rocksdb.ReadOptions) InvocationOnMock(org.mockito.invocation.InvocationOnMock) OneShotLatch(org.apache.flink.core.testutils.OneShotLatch) BlockerCheckpointStreamFactory(org.apache.flink.runtime.util.BlockerCheckpointStreamFactory) DummyEnvironment(org.apache.flink.runtime.operators.testutils.DummyEnvironment) Environment(org.apache.flink.runtime.execution.Environment) RocksObject(org.rocksdb.RocksObject) JobID(org.apache.flink.api.common.JobID)

Example 2 with Environment

use of org.apache.flink.runtime.execution.Environment in project flink by apache.

the class AsyncWaitOperatorTest method testTimeoutCleanup.

/**
	 * FLINK-5652
	 * Tests that registered timers are properly canceled upon completion of a
	 * {@link StreamRecordQueueEntry} in order to avoid resource leaks because TriggerTasks hold
	 * a reference on the StreamRecordQueueEntry.
	 */
@Test
public void testTimeoutCleanup() throws Exception {
    final Object lock = new Object();
    final long timeout = 100000L;
    final long timestamp = 1L;
    Environment environment = mock(Environment.class);
    when(environment.getMetricGroup()).thenReturn(new UnregisteredTaskMetricsGroup());
    when(environment.getTaskManagerInfo()).thenReturn(new TestingTaskManagerRuntimeInfo());
    when(environment.getUserClassLoader()).thenReturn(getClass().getClassLoader());
    when(environment.getTaskInfo()).thenReturn(new TaskInfo("testTask", 1, 0, 1, 0));
    ScheduledFuture<?> scheduledFuture = mock(ScheduledFuture.class);
    ProcessingTimeService processingTimeService = mock(ProcessingTimeService.class);
    when(processingTimeService.getCurrentProcessingTime()).thenReturn(timestamp);
    doReturn(scheduledFuture).when(processingTimeService).registerTimer(anyLong(), any(ProcessingTimeCallback.class));
    StreamTask<?, ?> containingTask = mock(StreamTask.class);
    when(containingTask.getEnvironment()).thenReturn(environment);
    when(containingTask.getCheckpointLock()).thenReturn(lock);
    when(containingTask.getProcessingTimeService()).thenReturn(processingTimeService);
    StreamConfig streamConfig = mock(StreamConfig.class);
    doReturn(IntSerializer.INSTANCE).when(streamConfig).getTypeSerializerIn1(any(ClassLoader.class));
    Output<StreamRecord<Integer>> output = mock(Output.class);
    AsyncWaitOperator<Integer, Integer> operator = new AsyncWaitOperator<>(new AsyncFunction<Integer, Integer>() {

        private static final long serialVersionUID = -3718276118074877073L;

        @Override
        public void asyncInvoke(Integer input, AsyncCollector<Integer> collector) throws Exception {
            collector.collect(Collections.singletonList(input));
        }
    }, timeout, 1, AsyncDataStream.OutputMode.UNORDERED);
    operator.setup(containingTask, streamConfig, output);
    operator.open();
    final StreamRecord<Integer> streamRecord = new StreamRecord<>(42, timestamp);
    synchronized (lock) {
        // processing an element will register a timeout
        operator.processElement(streamRecord);
    }
    synchronized (lock) {
        // closing the operator waits until all inputs have been processed
        operator.close();
    }
    // check that we actually outputted the result of the single input
    verify(output).collect(eq(streamRecord));
    verify(processingTimeService).registerTimer(eq(processingTimeService.getCurrentProcessingTime() + timeout), any(ProcessingTimeCallback.class));
    // check that we have cancelled our registered timeout
    verify(scheduledFuture).cancel(eq(true));
}
Also used : UnregisteredTaskMetricsGroup(org.apache.flink.runtime.operators.testutils.UnregisteredTaskMetricsGroup) StreamRecord(org.apache.flink.streaming.runtime.streamrecord.StreamRecord) StreamConfig(org.apache.flink.streaming.api.graph.StreamConfig) TimeoutException(java.util.concurrent.TimeoutException) ExecutionException(java.util.concurrent.ExecutionException) TaskInfo(org.apache.flink.api.common.TaskInfo) TestingTaskManagerRuntimeInfo(org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo) ProcessingTimeCallback(org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback) TestProcessingTimeService(org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService) ProcessingTimeService(org.apache.flink.streaming.runtime.tasks.ProcessingTimeService) Environment(org.apache.flink.runtime.execution.Environment) StreamExecutionEnvironment(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment) StreamMockEnvironment(org.apache.flink.streaming.runtime.tasks.StreamMockEnvironment) Test(org.junit.Test)

Example 3 with Environment

use of org.apache.flink.runtime.execution.Environment in project flink by apache.

the class AsyncWaitOperatorTest method testAsyncTimeout.

@Test
public void testAsyncTimeout() throws Exception {
    final long timeout = 10L;
    final AsyncWaitOperator<Integer, Integer> operator = new AsyncWaitOperator<>(new LazyAsyncFunction(), timeout, 2, AsyncDataStream.OutputMode.ORDERED);
    final Environment mockEnvironment = mock(Environment.class);
    final Configuration taskConfiguration = new Configuration();
    final ExecutionConfig executionConfig = new ExecutionConfig();
    final TaskMetricGroup metricGroup = new UnregisteredTaskMetricsGroup();
    final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();
    final TaskInfo taskInfo = new TaskInfo("foobarTask", 1, 0, 1, 1);
    when(mockEnvironment.getTaskConfiguration()).thenReturn(taskConfiguration);
    when(mockEnvironment.getExecutionConfig()).thenReturn(executionConfig);
    when(mockEnvironment.getMetricGroup()).thenReturn(metricGroup);
    when(mockEnvironment.getTaskManagerInfo()).thenReturn(taskManagerRuntimeInfo);
    when(mockEnvironment.getTaskInfo()).thenReturn(taskInfo);
    when(mockEnvironment.getUserClassLoader()).thenReturn(AsyncWaitOperatorTest.class.getClassLoader());
    final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator, IntSerializer.INSTANCE, mockEnvironment);
    final long initialTime = 0L;
    final ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    testHarness.setProcessingTime(initialTime);
    synchronized (testHarness.getCheckpointLock()) {
        testHarness.processElement(new StreamRecord<>(1, initialTime));
        testHarness.setProcessingTime(initialTime + 5L);
        testHarness.processElement(new StreamRecord<>(2, initialTime + 5L));
    }
    // trigger the timeout of the first stream record
    testHarness.setProcessingTime(initialTime + timeout + 1L);
    // allow the second async stream record to be processed
    LazyAsyncFunction.countDown();
    // wait until all async collectors in the buffer have been emitted out.
    synchronized (testHarness.getCheckpointLock()) {
        testHarness.close();
    }
    expectedOutput.add(new StreamRecord<>(2, initialTime + 5L));
    TestHarnessUtil.assertOutputEquals("Output with watermark was not correct.", expectedOutput, testHarness.getOutput());
    ArgumentCaptor<Throwable> argumentCaptor = ArgumentCaptor.forClass(Throwable.class);
    verify(mockEnvironment).failExternally(argumentCaptor.capture());
    Throwable failureCause = argumentCaptor.getValue();
    Assert.assertNotNull(failureCause.getCause());
    Assert.assertTrue(failureCause.getCause() instanceof ExecutionException);
    Assert.assertNotNull(failureCause.getCause().getCause());
    Assert.assertTrue(failureCause.getCause().getCause() instanceof TimeoutException);
}
Also used : UnregisteredTaskMetricsGroup(org.apache.flink.runtime.operators.testutils.UnregisteredTaskMetricsGroup) Configuration(org.apache.flink.configuration.Configuration) TestingTaskManagerRuntimeInfo(org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo) TaskManagerRuntimeInfo(org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo) TaskMetricGroup(org.apache.flink.runtime.metrics.groups.TaskMetricGroup) ExecutionConfig(org.apache.flink.api.common.ExecutionConfig) OneInputStreamOperatorTestHarness(org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness) TaskInfo(org.apache.flink.api.common.TaskInfo) TestingTaskManagerRuntimeInfo(org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo) Environment(org.apache.flink.runtime.execution.Environment) StreamExecutionEnvironment(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment) StreamMockEnvironment(org.apache.flink.streaming.runtime.tasks.StreamMockEnvironment) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) ExecutionException(java.util.concurrent.ExecutionException) TimeoutException(java.util.concurrent.TimeoutException) Test(org.junit.Test)

Example 4 with Environment

use of org.apache.flink.runtime.execution.Environment in project flink by apache.

the class StreamingRuntimeContextTest method createMockEnvironment.

private static Environment createMockEnvironment() {
    Environment env = mock(Environment.class);
    when(env.getUserClassLoader()).thenReturn(StreamingRuntimeContextTest.class.getClassLoader());
    when(env.getDistributedCacheEntries()).thenReturn(Collections.<String, Future<Path>>emptyMap());
    when(env.getTaskInfo()).thenReturn(new TaskInfo("test task", 1, 0, 1, 1));
    return env;
}
Also used : Path(org.apache.flink.core.fs.Path) TaskInfo(org.apache.flink.api.common.TaskInfo) DummyEnvironment(org.apache.flink.runtime.operators.testutils.DummyEnvironment) Environment(org.apache.flink.runtime.execution.Environment)

Example 5 with Environment

use of org.apache.flink.runtime.execution.Environment in project flink by apache.

the class StreamSourceOperatorTest method setupSourceOperator.

@SuppressWarnings("unchecked")
private static <T> void setupSourceOperator(StreamSource<T, ?> operator, TimeCharacteristic timeChar, long watermarkInterval, long latencyMarkInterval, final ProcessingTimeService timeProvider) {
    ExecutionConfig executionConfig = new ExecutionConfig();
    executionConfig.setAutoWatermarkInterval(watermarkInterval);
    executionConfig.setLatencyTrackingInterval(latencyMarkInterval);
    StreamConfig cfg = new StreamConfig(new Configuration());
    cfg.setStateBackend(new MemoryStateBackend());
    cfg.setTimeCharacteristic(timeChar);
    Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);
    StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class);
    when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE);
    StreamTask<?, ?> mockTask = mock(StreamTask.class);
    when(mockTask.getName()).thenReturn("Mock Task");
    when(mockTask.getCheckpointLock()).thenReturn(new Object());
    when(mockTask.getConfiguration()).thenReturn(cfg);
    when(mockTask.getEnvironment()).thenReturn(env);
    when(mockTask.getExecutionConfig()).thenReturn(executionConfig);
    when(mockTask.getAccumulatorMap()).thenReturn(Collections.<String, Accumulator<?, ?>>emptyMap());
    when(mockTask.getStreamStatusMaintainer()).thenReturn(streamStatusMaintainer);
    doAnswer(new Answer<ProcessingTimeService>() {

        @Override
        public ProcessingTimeService answer(InvocationOnMock invocation) throws Throwable {
            if (timeProvider == null) {
                throw new RuntimeException("The time provider is null.");
            }
            return timeProvider;
        }
    }).when(mockTask).getProcessingTimeService();
    operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
}
Also used : StreamRecord(org.apache.flink.streaming.runtime.streamrecord.StreamRecord) Configuration(org.apache.flink.configuration.Configuration) MemoryStateBackend(org.apache.flink.runtime.state.memory.MemoryStateBackend) StreamConfig(org.apache.flink.streaming.api.graph.StreamConfig) StreamStatusMaintainer(org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer) DummyEnvironment(org.apache.flink.runtime.operators.testutils.DummyEnvironment) ExecutionConfig(org.apache.flink.api.common.ExecutionConfig) ProcessingTimeService(org.apache.flink.streaming.runtime.tasks.ProcessingTimeService) TestProcessingTimeService(org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService) InvocationOnMock(org.mockito.invocation.InvocationOnMock) DummyEnvironment(org.apache.flink.runtime.operators.testutils.DummyEnvironment) Environment(org.apache.flink.runtime.execution.Environment)

Aggregations

Environment (org.apache.flink.runtime.execution.Environment)33 Configuration (org.apache.flink.configuration.Configuration)15 ExecutionConfig (org.apache.flink.api.common.ExecutionConfig)14 Test (org.junit.Test)13 TestingTaskManagerRuntimeInfo (org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo)9 TaskInfo (org.apache.flink.api.common.TaskInfo)8 StreamConfig (org.apache.flink.streaming.api.graph.StreamConfig)8 IOException (java.io.IOException)6 DummyEnvironment (org.apache.flink.runtime.operators.testutils.DummyEnvironment)6 MockEnvironment (org.apache.flink.runtime.operators.testutils.MockEnvironment)6 StreamExecutionEnvironment (org.apache.flink.streaming.api.environment.StreamExecutionEnvironment)6 StreamRecord (org.apache.flink.streaming.runtime.streamrecord.StreamRecord)6 JobID (org.apache.flink.api.common.JobID)5 UnregisteredTaskMetricsGroup (org.apache.flink.runtime.operators.testutils.UnregisteredTaskMetricsGroup)5 CloseableRegistry (org.apache.flink.core.fs.CloseableRegistry)4 MemoryStateBackend (org.apache.flink.runtime.state.memory.MemoryStateBackend)4 TestProcessingTimeService (org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService)4 ExecutionException (java.util.concurrent.ExecutionException)3 RejectedExecutionException (java.util.concurrent.RejectedExecutionException)3 CancelTaskException (org.apache.flink.runtime.execution.CancelTaskException)3