Examples with StreamConfig org.apache.flink.streaming.api.graph.StreamConfig used on opensource projects

Example 71 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class SubtaskCheckpointCoordinatorTest method testBroadcastCancelCheckpointMarkerOnAbortingFromCoordinator.

@Test
public void testBroadcastCancelCheckpointMarkerOnAbortingFromCoordinator() throws Exception {
    OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, 1, 1, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    streamConfig.setStreamOperator(new MapOperator());
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    MockEnvironment mockEnvironment = MockEnvironment.builder().build();
    try (SubtaskCheckpointCoordinator subtaskCheckpointCoordinator = new MockSubtaskCheckpointCoordinatorBuilder().setEnvironment(mockEnvironment).build()) {
        ArrayList<Object> recordOrEvents = new ArrayList<>();
        StreamElementSerializer<String> stringStreamElementSerializer = new StreamElementSerializer<>(StringSerializer.INSTANCE);
        ResultPartitionWriter resultPartitionWriter = new RecordOrEventCollectingResultPartitionWriter<>(recordOrEvents, stringStreamElementSerializer);
        mockEnvironment.addOutputs(Collections.singletonList(resultPartitionWriter));
        OneInputStreamTask<String, String> task = testHarness.getTask();
        OperatorChain<String, OneInputStreamOperator<String, String>> operatorChain = new RegularOperatorChain<>(task, StreamTask.createRecordWriterDelegate(streamConfig, mockEnvironment));
        long checkpointId = 42L;
        // notify checkpoint aborted before execution.
        subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true);
        subtaskCheckpointCoordinator.checkpointState(new CheckpointMetaData(checkpointId, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetricsBuilder(), operatorChain, false, () -> false);
        assertEquals(1, recordOrEvents.size());
        Object recordOrEvent = recordOrEvents.get(0);
        // ensure CancelCheckpointMarker is broadcast downstream.
        assertTrue(recordOrEvent instanceof CancelCheckpointMarker);
        assertEquals(checkpointId, ((CancelCheckpointMarker) recordOrEvent).getCheckpointId());
        testHarness.endInput();
        testHarness.waitForTaskCompletion();
    }
}

Example 72 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StreamTaskSelectiveReadingTest method testBase.

private void testBase(TwoInputStreamOperator<String, Integer, String> streamOperator, boolean prepareDataBeforeProcessing, ConcurrentLinkedQueue<Object> expectedOutput, boolean orderedCheck) throws Exception {
    final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness = new TwoInputStreamTaskTestHarness<>(TestSelectiveReadingTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    streamConfig.setStreamOperator(streamOperator);
    streamConfig.setOperatorID(new OperatorID());
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    boolean isProcessing = false;
    if (!prepareDataBeforeProcessing) {
        ((TestSelectiveReadingTask) testHarness.getTask()).startProcessing();
        isProcessing = true;
    }
    testHarness.processElement(new StreamRecord<>("Hello-1"), 0, 0);
    // wait until the input is processed to test the listening and blocking logic
    if (!prepareDataBeforeProcessing) {
        testHarness.waitForInputProcessing();
    }
    testHarness.processElement(new StreamRecord<>("Hello-2"), 0, 0);
    testHarness.processElement(new StreamRecord<>("Hello-3"), 0, 0);
    testHarness.processElement(new StreamRecord<>(1), 1, 0);
    testHarness.processElement(new StreamRecord<>(2), 1, 0);
    testHarness.processElement(new StreamRecord<>(3), 1, 0);
    testHarness.processElement(new StreamRecord<>(4), 1, 0);
    testHarness.endInput();
    if (!isProcessing) {
        ((TestSelectiveReadingTask) testHarness.getTask()).startProcessing();
    }
    testHarness.waitForTaskCompletion(10_000L);
    LinkedBlockingQueue<Object> output = testHarness.getOutput();
    if (orderedCheck) {
        TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, output);
    } else {
        String[] expectedResult = expectedOutput.stream().map(StreamTaskSelectiveReadingTest::elementToString).toArray(String[]::new);
        Arrays.sort(expectedResult);
        String[] result = output.stream().map(StreamTaskSelectiveReadingTest::elementToString).toArray(String[]::new);
        Arrays.sort(result);
        assertArrayEquals("Output was not correct.", expectedResult, result);
    }
}

Example 73 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StreamConfigChainer method chain.

public <IN, OUT> StreamConfigChainer<OWNER> chain(OperatorID operatorID, StreamOperatorFactory<OUT> operatorFactory, TypeSerializer<IN> inputSerializer, TypeSerializer<OUT> outputSerializer, boolean createKeyedStateBackend) {
    chainIndex++;
    StreamEdge streamEdge = new StreamEdge(new StreamNode(tailConfig.getChainIndex(), null, null, (StreamOperator<?>) null, null, null), new StreamNode(chainIndex, null, null, (StreamOperator<?>) null, null, null), 0, null, null);
    streamEdge.setBufferTimeout(bufferTimeout);
    tailConfig.setChainedOutputs(Collections.singletonList(streamEdge));
    tailConfig = new StreamConfig(new Configuration());
    tailConfig.setStreamOperatorFactory(checkNotNull(operatorFactory));
    tailConfig.setOperatorID(checkNotNull(operatorID));
    tailConfig.setupNetworkInputs(inputSerializer);
    tailConfig.setTypeSerializerOut(outputSerializer);
    if (createKeyedStateBackend) {
        // used to test multiple stateful operators chained in a single task.
        tailConfig.setStateKeySerializer(inputSerializer);
        tailConfig.setStateBackendUsesManagedMemory(true);
        tailConfig.setManagedMemoryFractionOperatorOfUseCase(ManagedMemoryUseCase.STATE_BACKEND, 1.0);
    }
    tailConfig.setChainIndex(chainIndex);
    chainedConfigs.put(chainIndex, tailConfig);
    return this;
}

Example 74 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class TaskCheckpointingBehaviourTest method createTask.

// ------------------------------------------------------------------------
// Utilities
// ------------------------------------------------------------------------
private static Task createTask(StreamOperator<?> op, StateBackend backend, CheckpointResponder checkpointResponder) throws IOException {
    Configuration taskConfig = new Configuration();
    StreamConfig cfg = new StreamConfig(taskConfig);
    cfg.setStreamOperator(op);
    cfg.setOperatorID(new OperatorID());
    cfg.setStateBackend(backend);
    ExecutionConfig executionConfig = new ExecutionConfig();
    JobInformation jobInformation = new JobInformation(new JobID(), "test job name", new SerializedValue<>(executionConfig), new Configuration(), Collections.emptyList(), Collections.emptyList());
    TaskInformation taskInformation = new TaskInformation(new JobVertexID(), "test task name", 1, 11, TestStreamTask.class.getName(), taskConfig);
    ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
    return new Task(jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0, 0, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), mock(MemoryManager.class), mock(IOManager.class), shuffleEnvironment, new KvStateService(new KvStateRegistry(), null, null), mock(BroadcastVariableManager.class), new TaskEventDispatcher(), ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES, new TestTaskStateManager(), mock(TaskManagerActions.class), mock(InputSplitProvider.class), checkpointResponder, new NoOpTaskOperatorEventGateway(), new TestGlobalAggregateManager(), TestingClassLoaderLease.newBuilder().build(), new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() }, VoidPermanentBlobService.INSTANCE), new TestingTaskManagerRuntimeInfo(), UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(), new NoOpResultPartitionConsumableNotifier(), mock(PartitionProducerStateChecker.class), Executors.directExecutor());
}

Example 75 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class TwoInputStreamTaskTest method testCheckpointBarriers.

/**
 * This test verifies that checkpoint barriers are correctly forwarded.
 */
@Test
public void testCheckpointBarriers() throws Exception {
    final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness = new TwoInputStreamTaskTestHarness<>(TwoInputStreamTask::new, 2, 2, new int[] { 1, 2 }, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    CoStreamMap<String, Integer, String> coMapOperator = new CoStreamMap<>(new IdentityMap());
    streamConfig.setStreamOperator(coMapOperator);
    streamConfig.setOperatorID(new OperatorID());
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    long initialTime = 0L;
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0);
    // This one should go through
    testHarness.processElement(new StreamRecord<>("Ciao-0-0", initialTime), 0, 1);
    expectedOutput.add(new StreamRecord<>("Ciao-0-0", initialTime));
    testHarness.waitForInputProcessing();
    // These elements should be forwarded, since we did not yet receive a checkpoint barrier
    // on that input, only add to same input, otherwise we would not know the ordering
    // of the output since the Task might read the inputs in any order
    testHarness.processElement(new StreamRecord<>(11, initialTime), 1, 1);
    testHarness.processElement(new StreamRecord<>(111, initialTime), 1, 1);
    expectedOutput.add(new StreamRecord<>("11", initialTime));
    expectedOutput.add(new StreamRecord<>("111", initialTime));
    testHarness.waitForInputProcessing();
    // TODO Use count down latches instead as a cleaner solution
    for (int i = 0; i < 20; ++i) {
        if (testHarness.getOutput().size() >= expectedOutput.size()) {
            break;
        } else {
            Thread.sleep(100);
        }
    }
    // we should not yet see the barrier, only the two elements from non-blocked input
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1);
    testHarness.waitForInputProcessing();
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    // now we should see the barrier
    expectedOutput.add(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
    Assert.assertEquals(3, resultElements.size());
}