Examples with StreamConfig org.apache.flink.streaming.api.graph.StreamConfig used on opensource projects

Example 16 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class AsyncWaitOperatorTest method testStateSnapshotAndRestore.

@Test
public void testStateSnapshotAndRestore() throws Exception {
    final OneInputStreamTaskTestHarness<Integer, Integer> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, 1, 1, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    AsyncWaitOperatorFactory<Integer, Integer> factory = new AsyncWaitOperatorFactory<>(new LazyAsyncFunction(), TIMEOUT, 4, AsyncDataStream.OutputMode.ORDERED);
    final StreamConfig streamConfig = testHarness.getStreamConfig();
    OperatorID operatorID = new OperatorID(42L, 4711L);
    streamConfig.setStreamOperatorFactory(factory);
    streamConfig.setOperatorID(operatorID);
    final TestTaskStateManager taskStateManagerMock = testHarness.getTaskStateManager();
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    final OneInputStreamTask<Integer, Integer> task = testHarness.getTask();
    final long initialTime = 0L;
    testHarness.processElement(new StreamRecord<>(1, initialTime + 1));
    testHarness.processElement(new StreamRecord<>(2, initialTime + 2));
    testHarness.processElement(new StreamRecord<>(3, initialTime + 3));
    testHarness.processElement(new StreamRecord<>(4, initialTime + 4));
    testHarness.waitForInputProcessing();
    final long checkpointId = 1L;
    final long checkpointTimestamp = 1L;
    final CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, checkpointTimestamp);
    task.triggerCheckpointAsync(checkpointMetaData, CheckpointOptions.forCheckpointWithDefaultLocation());
    taskStateManagerMock.getWaitForReportLatch().await();
    assertEquals(checkpointId, taskStateManagerMock.getReportedCheckpointId());
    LazyAsyncFunction.countDown();
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    // set the operator state from previous attempt into the restored one
    TaskStateSnapshot subtaskStates = taskStateManagerMock.getLastJobManagerTaskStateSnapshot();
    final OneInputStreamTaskTestHarness<Integer, Integer> restoredTaskHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);
    restoredTaskHarness.setTaskStateSnapshot(checkpointId, subtaskStates);
    restoredTaskHarness.setupOutputForSingletonOperatorChain();
    AsyncWaitOperatorFactory<Integer, Integer> restoredOperator = new AsyncWaitOperatorFactory<>(new MyAsyncFunction(), TIMEOUT, 6, AsyncDataStream.OutputMode.ORDERED);
    restoredTaskHarness.getStreamConfig().setStreamOperatorFactory(restoredOperator);
    restoredTaskHarness.getStreamConfig().setOperatorID(operatorID);
    restoredTaskHarness.invoke();
    restoredTaskHarness.waitForTaskRunning();
    final OneInputStreamTask<Integer, Integer> restoredTask = restoredTaskHarness.getTask();
    restoredTaskHarness.processElement(new StreamRecord<>(5, initialTime + 5));
    restoredTaskHarness.processElement(new StreamRecord<>(6, initialTime + 6));
    restoredTaskHarness.processElement(new StreamRecord<>(7, initialTime + 7));
    // trigger the checkpoint while processing stream elements
    restoredTask.triggerCheckpointAsync(new CheckpointMetaData(checkpointId, checkpointTimestamp), CheckpointOptions.forCheckpointWithDefaultLocation()).get();
    restoredTaskHarness.processElement(new StreamRecord<>(8, initialTime + 8));
    restoredTaskHarness.endInput();
    restoredTaskHarness.waitForTaskCompletion();
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    expectedOutput.add(new StreamRecord<>(2, initialTime + 1));
    expectedOutput.add(new StreamRecord<>(4, initialTime + 2));
    expectedOutput.add(new StreamRecord<>(6, initialTime + 3));
    expectedOutput.add(new StreamRecord<>(8, initialTime + 4));
    expectedOutput.add(new StreamRecord<>(10, initialTime + 5));
    expectedOutput.add(new StreamRecord<>(12, initialTime + 6));
    expectedOutput.add(new StreamRecord<>(14, initialTime + 7));
    expectedOutput.add(new StreamRecord<>(16, initialTime + 8));
    // remove CheckpointBarrier which is not expected
    restoredTaskHarness.getOutput().removeIf(record -> record instanceof CheckpointBarrier);
    TestHarnessUtil.assertOutputEquals("StateAndRestored Test Output was not correct.", expectedOutput, restoredTaskHarness.getOutput());
}

Example 17 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class AsyncWaitOperatorTest method testOperatorChainWithProcessingTime.

/**
 * Tests that the AsyncWaitOperator works together with chaining.
 */
@Test
public void testOperatorChainWithProcessingTime() throws Exception {
    JobVertex chainedVertex = createChainedVertex(new MyAsyncFunction(), new MyAsyncFunction());
    final OneInputStreamTaskTestHarness<Integer, Integer> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, 1, 1, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    testHarness.taskConfig = chainedVertex.getConfiguration();
    final StreamConfig streamConfig = testHarness.getStreamConfig();
    final StreamConfig operatorChainStreamConfig = new StreamConfig(chainedVertex.getConfiguration());
    streamConfig.setStreamOperatorFactory(operatorChainStreamConfig.getStreamOperatorFactory(AsyncWaitOperatorTest.class.getClassLoader()));
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    long initialTimestamp = 0L;
    testHarness.processElement(new StreamRecord<>(5, initialTimestamp));
    testHarness.processElement(new StreamRecord<>(6, initialTimestamp + 1L));
    testHarness.processElement(new StreamRecord<>(7, initialTimestamp + 2L));
    testHarness.processElement(new StreamRecord<>(8, initialTimestamp + 3L));
    testHarness.processElement(new StreamRecord<>(9, initialTimestamp + 4L));
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    List<Object> expectedOutput = new LinkedList<>();
    expectedOutput.add(new StreamRecord<>(22, initialTimestamp));
    expectedOutput.add(new StreamRecord<>(26, initialTimestamp + 1L));
    expectedOutput.add(new StreamRecord<>(30, initialTimestamp + 2L));
    expectedOutput.add(new StreamRecord<>(34, initialTimestamp + 3L));
    expectedOutput.add(new StreamRecord<>(38, initialTimestamp + 4L));
    TestHarnessUtil.assertOutputEqualsSorted("Test for chained operator with AsyncWaitOperator failed", expectedOutput, testHarness.getOutput(), new StreamRecordComparator());
}

Example 18 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class BufferDataOverWindowOperatorTest method test.

private void test(OverWindowFrame[] frames, GenericRowData[] expect) throws Exception {
    MockEnvironment env = new MockEnvironmentBuilder().setIOManager(ioManager).setMemoryManager(memoryManager).build();
    StreamTask<Object, StreamOperator<Object>> task = new StreamTask<Object, StreamOperator<Object>>(env) {

        @Override
        protected void init() {
        }
    };
    operator = new BufferDataOverWindowOperator(frames, comparator, true) {

        {
            output = new NonBufferOverWindowOperatorTest.ConsumerOutput(new Consumer<RowData>() {

                @Override
                public void accept(RowData r) {
                    collect.add(GenericRowData.of(r.getInt(0), r.getLong(1), r.getLong(2), r.getLong(3), r.getLong(4)));
                }
            });
        }

        @Override
        public ClassLoader getUserCodeClassloader() {
            return Thread.currentThread().getContextClassLoader();
        }

        @Override
        public StreamConfig getOperatorConfig() {
            StreamConfig conf = mock(StreamConfig.class);
            when(conf.<RowData>getTypeSerializerIn1(getUserCodeClassloader())).thenReturn(inputSer);
            when(conf.getManagedMemoryFractionOperatorUseCaseOfSlot(eq(ManagedMemoryUseCase.OPERATOR), any(Configuration.class), any(ClassLoader.class))).thenReturn(0.99);
            return conf;
        }

        @Override
        public StreamTask<?, ?> getContainingTask() {
            return task;
        }

        @Override
        public StreamingRuntimeContext getRuntimeContext() {
            return mock(StreamingRuntimeContext.class);
        }
    };
    operator.setProcessingTimeService(new TestProcessingTimeService());
    operator.open();
    addRow(0, 1L, 4L);
    /* 1 **/
    addRow(0, 1L, 1L);
    /* 2 **/
    addRow(0, 1L, 1L);
    /* 3 **/
    addRow(0, 1L, 1L);
    /* 4 **/
    addRow(1, 5L, 2L);
    /* 5 **/
    addRow(2, 5L, 4L);
    /* 6 **/
    addRow(2, 6L, 2L);
    /* 7 **/
    addRow(2, 6L, 2L);
    /* 8 **/
    addRow(2, 6L, 2L);
    /* 9 **/
    operator.endInput();
    GenericRowData[] outputs = this.collect.toArray(new GenericRowData[0]);
    Assert.assertArrayEquals(expect, outputs);
    operator.close();
}

Example 19 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class ForwardForConsecutiveHashPartitionerTest method testConvertToForwardPartitioner.

private void testConvertToForwardPartitioner(StreamExchangeMode streamExchangeMode) {
    JobGraph jobGraph = StreamPartitionerTestUtils.createJobGraph("group1", "group1", new ForwardForConsecutiveHashPartitioner<>(new KeyGroupStreamPartitioner<>(record -> 0L, 100)));
    List<JobVertex> jobVertices = jobGraph.getVerticesSortedTopologicallyFromSources();
    assertThat(jobVertices.size(), is(1));
    JobVertex vertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(0);
    StreamConfig sourceConfig = new StreamConfig(vertex.getConfiguration());
    StreamEdge edge = sourceConfig.getChainedOutputs(getClass().getClassLoader()).get(0);
    assertThat(edge.getPartitioner(), instanceOf(ForwardPartitioner.class));
}

Example 20 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class ForwardForConsecutiveHashPartitionerTest method testConvertToHashPartitioner.

private void testConvertToHashPartitioner(StreamExchangeMode streamExchangeMode) {
    JobGraph jobGraph = StreamPartitionerTestUtils.createJobGraph("group1", "group2", new ForwardForConsecutiveHashPartitioner<>(new KeyGroupStreamPartitioner<>(record -> 0L, 100)));
    List<JobVertex> jobVertices = jobGraph.getVerticesSortedTopologicallyFromSources();
    assertThat(jobVertices.size(), is(2));
    JobVertex sourceVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(0);
    StreamConfig sourceConfig = new StreamConfig(sourceVertex.getConfiguration());
    StreamEdge edge = sourceConfig.getNonChainedOutputs(getClass().getClassLoader()).get(0);
    assertThat(edge.getPartitioner(), instanceOf(KeyGroupStreamPartitioner.class));
}