Examples with Configuration org.apache.flink.configuration.Configuration used on opensource projects

Example 91 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class InputFormatSourceFunctionTest method testFormatLifecycle.

private void testFormatLifecycle(final boolean midCancel) throws Exception {
    final int noOfSplits = 5;
    final int cancelAt = 2;
    final LifeCycleTestInputFormat format = new LifeCycleTestInputFormat();
    final InputFormatSourceFunction<Integer> reader = new InputFormatSourceFunction<>(format, TypeInformation.of(Integer.class));
    reader.setRuntimeContext(new MockRuntimeContext(format, noOfSplits));
    Assert.assertTrue(!format.isConfigured);
    Assert.assertTrue(!format.isInputFormatOpen);
    Assert.assertTrue(!format.isSplitOpen);
    reader.open(new Configuration());
    Assert.assertTrue(format.isConfigured);
    TestSourceContext ctx = new TestSourceContext(reader, format, midCancel, cancelAt);
    reader.run(ctx);
    int splitsSeen = ctx.getSplitsSeen();
    Assert.assertTrue(midCancel ? splitsSeen == cancelAt : splitsSeen == noOfSplits);
    // we have exhausted the splits so the
    // format and splits should be closed by now
    Assert.assertTrue(!format.isSplitOpen);
    Assert.assertTrue(!format.isInputFormatOpen);
}

Example 92 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class AsyncWaitOperatorTest method createChainedVertex.

private JobVertex createChainedVertex(boolean withLazyFunction) {
    StreamExecutionEnvironment chainEnv = StreamExecutionEnvironment.getExecutionEnvironment();
    // the input is only used to construct a chained operator, and they will not be used in the real tests.
    DataStream<Integer> input = chainEnv.fromElements(1, 2, 3);
    if (withLazyFunction) {
        input = AsyncDataStream.orderedWait(input, new LazyAsyncFunction(), TIMEOUT, TimeUnit.MILLISECONDS, 6);
    } else {
        input = AsyncDataStream.orderedWait(input, new MyAsyncFunction(), TIMEOUT, TimeUnit.MILLISECONDS, 6);
    }
    // the map function is designed to chain after async function. we place an Integer object in it and
    // it is initialized in the open() method.
    // it is used to verify that operators in the operator chain should be opened from the tail to the head,
    // so the result from AsyncWaitOperator can pass down successfully and correctly.
    // if not, the test can not be passed.
    input = input.map(new RichMapFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        private Integer initialValue = null;

        @Override
        public void open(Configuration parameters) throws Exception {
            initialValue = 1;
        }

        @Override
        public Integer map(Integer value) throws Exception {
            return initialValue + value;
        }
    });
    input = AsyncDataStream.unorderedWait(input, new MyAsyncFunction(), TIMEOUT, TimeUnit.MILLISECONDS, 3);
    input.map(new MapFunction<Integer, Integer>() {

        private static final long serialVersionUID = 5162085254238405527L;

        @Override
        public Integer map(Integer value) throws Exception {
            return value;
        }
    }).startNewChain().addSink(new DiscardingSink<Integer>());
    // be build our own OperatorChain
    final JobGraph jobGraph = chainEnv.getStreamGraph().getJobGraph();
    Assert.assertTrue(jobGraph.getVerticesSortedTopologicallyFromSources().size() == 3);
    return jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
}

Example 93 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class AsyncWaitOperatorTest method testAsyncTimeout.

@Test
public void testAsyncTimeout() throws Exception {
    final long timeout = 10L;
    final AsyncWaitOperator<Integer, Integer> operator = new AsyncWaitOperator<>(new LazyAsyncFunction(), timeout, 2, AsyncDataStream.OutputMode.ORDERED);
    final Environment mockEnvironment = mock(Environment.class);
    final Configuration taskConfiguration = new Configuration();
    final ExecutionConfig executionConfig = new ExecutionConfig();
    final TaskMetricGroup metricGroup = new UnregisteredTaskMetricsGroup();
    final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();
    final TaskInfo taskInfo = new TaskInfo("foobarTask", 1, 0, 1, 1);
    when(mockEnvironment.getTaskConfiguration()).thenReturn(taskConfiguration);
    when(mockEnvironment.getExecutionConfig()).thenReturn(executionConfig);
    when(mockEnvironment.getMetricGroup()).thenReturn(metricGroup);
    when(mockEnvironment.getTaskManagerInfo()).thenReturn(taskManagerRuntimeInfo);
    when(mockEnvironment.getTaskInfo()).thenReturn(taskInfo);
    when(mockEnvironment.getUserClassLoader()).thenReturn(AsyncWaitOperatorTest.class.getClassLoader());
    final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator, IntSerializer.INSTANCE, mockEnvironment);
    final long initialTime = 0L;
    final ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    testHarness.setProcessingTime(initialTime);
    synchronized (testHarness.getCheckpointLock()) {
        testHarness.processElement(new StreamRecord<>(1, initialTime));
        testHarness.setProcessingTime(initialTime + 5L);
        testHarness.processElement(new StreamRecord<>(2, initialTime + 5L));
    }
    // trigger the timeout of the first stream record
    testHarness.setProcessingTime(initialTime + timeout + 1L);
    // allow the second async stream record to be processed
    LazyAsyncFunction.countDown();
    // wait until all async collectors in the buffer have been emitted out.
    synchronized (testHarness.getCheckpointLock()) {
        testHarness.close();
    }
    expectedOutput.add(new StreamRecord<>(2, initialTime + 5L));
    TestHarnessUtil.assertOutputEquals("Output with watermark was not correct.", expectedOutput, testHarness.getOutput());
    ArgumentCaptor<Throwable> argumentCaptor = ArgumentCaptor.forClass(Throwable.class);
    verify(mockEnvironment).failExternally(argumentCaptor.capture());
    Throwable failureCause = argumentCaptor.getValue();
    Assert.assertNotNull(failureCause.getCause());
    Assert.assertTrue(failureCause.getCause() instanceof ExecutionException);
    Assert.assertNotNull(failureCause.getCause().getCause());
    Assert.assertTrue(failureCause.getCause().getCause() instanceof TimeoutException);
}

Example 94 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class InternalWindowFunctionTest method testInternalSingleValueProcessWindowFunction.

@SuppressWarnings("unchecked")
@Test
public void testInternalSingleValueProcessWindowFunction() throws Exception {
    ProcessWindowFunctionMock mock = mock(ProcessWindowFunctionMock.class);
    InternalSingleValueProcessWindowFunction<Long, String, Long, TimeWindow> windowFunction = new InternalSingleValueProcessWindowFunction<>(mock);
    // check setOutputType
    TypeInformation<String> stringType = BasicTypeInfo.STRING_TYPE_INFO;
    ExecutionConfig execConf = new ExecutionConfig();
    execConf.setParallelism(42);
    StreamingFunctionUtils.setOutputType(windowFunction, stringType, execConf);
    verify(mock).setOutputType(stringType, execConf);
    // check open
    Configuration config = new Configuration();
    windowFunction.open(config);
    verify(mock).open(config);
    // check setRuntimeContext
    RuntimeContext rCtx = mock(RuntimeContext.class);
    windowFunction.setRuntimeContext(rCtx);
    verify(mock).setRuntimeContext(rCtx);
    // check apply
    TimeWindow w = mock(TimeWindow.class);
    Collector<String> c = (Collector<String>) mock(Collector.class);
    windowFunction.apply(42L, w, 23L, c);
    verify(mock).process(eq(42L), (ProcessWindowFunctionMock.Context) anyObject(), (Iterable<Long>) argThat(IsIterableContainingInOrder.contains(23L)), eq(c));
    // check close
    windowFunction.close();
    verify(mock).close();
}

Example 95 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class InternalWindowFunctionTest method testInternalIterableAllWindowFunction.

@SuppressWarnings("unchecked")
@Test
public void testInternalIterableAllWindowFunction() throws Exception {
    AllWindowFunctionMock mock = mock(AllWindowFunctionMock.class);
    InternalIterableAllWindowFunction<Long, String, TimeWindow> windowFunction = new InternalIterableAllWindowFunction<>(mock);
    // check setOutputType
    TypeInformation<String> stringType = BasicTypeInfo.STRING_TYPE_INFO;
    ExecutionConfig execConf = new ExecutionConfig();
    execConf.setParallelism(42);
    StreamingFunctionUtils.setOutputType(windowFunction, stringType, execConf);
    verify(mock).setOutputType(stringType, execConf);
    // check open
    Configuration config = new Configuration();
    windowFunction.open(config);
    verify(mock).open(config);
    // check setRuntimeContext
    RuntimeContext rCtx = mock(RuntimeContext.class);
    windowFunction.setRuntimeContext(rCtx);
    verify(mock).setRuntimeContext(rCtx);
    // check apply
    TimeWindow w = mock(TimeWindow.class);
    Iterable<Long> i = (Iterable<Long>) mock(Iterable.class);
    Collector<String> c = (Collector<String>) mock(Collector.class);
    windowFunction.apply(((byte) 0), w, i, c);
    verify(mock).apply(w, i, c);
    // check close
    windowFunction.close();
    verify(mock).close();
}