Examples with StreamConfig org.apache.flink.streaming.api.graph.StreamConfig used on opensource projects

Example 86 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class SourceStreamTaskTest method testNotMarkingEndOfInputWhenTaskCancelled.

@Test
public void testNotMarkingEndOfInputWhenTaskCancelled() throws Exception {
    final StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>(SourceStreamTask::new, STRING_TYPE_INFO);
    testHarness.setupOperatorChain(new OperatorID(), new StreamSource<>(new CancelTestSource(STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), "Hello"))).chain(new OperatorID(), new TestBoundedOneInputStreamOperator("Operator1"), STRING_TYPE_INFO.createSerializer(new ExecutionConfig())).finish();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    streamConfig.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.invoke();
    CancelTestSource.getDataProcessing().await();
    testHarness.getTask().cancel();
    try {
        testHarness.waitForTaskCompletion();
    } catch (Throwable t) {
        if (!ExceptionUtils.findThrowable(t, CancelTaskException.class).isPresent()) {
            throw t;
        }
    }
    expectedOutput.add(new StreamRecord<>("Hello"));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}

Example 87 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class SourceStreamTaskTest method testCheckpointing.

/**
 * This test ensures that the SourceStreamTask properly serializes checkpointing and element
 * emission. This also verifies that there are no concurrent invocations of the checkpoint
 * method on the source operator.
 *
 * <p>The source emits elements and performs checkpoints. We have several checkpointer threads
 * that fire checkpoint requests at the source task.
 *
 * <p>If element emission and checkpointing are not in series the count of elements at the
 * beginning of a checkpoint and at the end of a checkpoint are not the same because the source
 * kept emitting elements while the checkpoint was ongoing.
 */
@Test
@SuppressWarnings("unchecked")
public void testCheckpointing() throws Exception {
    final int numElements = 100;
    final int numCheckpoints = 100;
    final int numCheckpointers = 1;
    // in ms
    final int checkpointInterval = 5;
    final int sourceCheckpointDelay = // how many random values we sum up in storeCheckpoint
    1000;
    // in ms
    final int sourceReadDelay = 1;
    ExecutorService executor = Executors.newFixedThreadPool(10);
    try {
        final TupleTypeInfo<Tuple2<Long, Integer>> typeInfo = new TupleTypeInfo<>(BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);
        final StreamTaskTestHarness<Tuple2<Long, Integer>> testHarness = new StreamTaskTestHarness<>(SourceStreamTask::new, typeInfo);
        testHarness.setupOutputForSingletonOperatorChain();
        StreamConfig streamConfig = testHarness.getStreamConfig();
        StreamSource<Tuple2<Long, Integer>, ?> sourceOperator = new StreamSource<>(new MockSource(numElements, sourceCheckpointDelay, sourceReadDelay));
        streamConfig.setStreamOperator(sourceOperator);
        streamConfig.setOperatorID(new OperatorID());
        // prepare the
        Future<Boolean>[] checkpointerResults = new Future[numCheckpointers];
        // invoke this first, so the tasks are actually running when the checkpoints are
        // scheduled
        testHarness.invoke();
        testHarness.waitForTaskRunning();
        final StreamTask<Tuple2<Long, Integer>, ?> sourceTask = testHarness.getTask();
        for (int i = 0; i < numCheckpointers; i++) {
            checkpointerResults[i] = executor.submit(new Checkpointer(numCheckpoints, checkpointInterval, sourceTask));
        }
        testHarness.waitForTaskCompletion();
        // will be rethrown here
        for (int i = 0; i < numCheckpointers; i++) {
            if (!checkpointerResults[i].isDone()) {
                checkpointerResults[i].cancel(true);
            }
            if (!checkpointerResults[i].isCancelled()) {
                checkpointerResults[i].get();
            }
        }
        List<Tuple2<Long, Integer>> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
        Assert.assertEquals(numElements, resultElements.size());
    } finally {
        executor.shutdown();
    }
}

Example 88 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class SourceStreamTaskTest method testCancellationWithSourceBlockedOnLock.

/**
 * Note that this test is testing also for the shared cancellation logic inside {@link
 * StreamTask} which, as of the time this test is being written, is not tested anywhere else
 * (like {@link StreamTaskTest} or {@link OneInputStreamTaskTest}).
 */
public void testCancellationWithSourceBlockedOnLock(boolean withPendingMail, boolean throwInCancel) throws Exception {
    final StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>(SourceStreamTask::new, STRING_TYPE_INFO);
    CancelLockingSource.reset();
    testHarness.setupOperatorChain(new OperatorID(), new StreamSource<>(new CancelLockingSource(throwInCancel))).chain(new OperatorID(), new TestBoundedOneInputStreamOperator("Operator1"), STRING_TYPE_INFO.createSerializer(new ExecutionConfig())).finish();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    streamConfig.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    testHarness.invoke();
    CancelLockingSource.awaitRunning();
    if (withPendingMail) {
        // This pending mail should be blocked on checkpointLock acquisition, blocking the
        // mailbox (task) thread.
        testHarness.getTask().getMailboxExecutorFactory().createExecutor(0).execute(() -> assertFalse("This should never execute before task cancelation", testHarness.getTask().isRunning()), "Test");
    }
    try {
        testHarness.getTask().cancel();
    } catch (ExpectedTestException e) {
        checkState(throwInCancel);
    }
    try {
        testHarness.waitForTaskCompletion();
    } catch (Throwable t) {
        if (!ExceptionUtils.findThrowable(t, InterruptedException.class).isPresent() && !ExceptionUtils.findThrowable(t, CancelTaskException.class).isPresent()) {
            throw t;
        }
    }
}

Example 89 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StreamTaskCancellationBarrierTest method testDeclineCallOnCancelBarrierOneInput.

/**
 * This test verifies (for onw input tasks) that the Stream tasks react the following way to
 * receiving a checkpoint cancellation barrier: - send a "decline checkpoint" notification out
 * (to the JobManager) - emit a cancellation barrier downstream.
 */
@Test
public void testDeclineCallOnCancelBarrierOneInput() throws Exception {
    OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, 1, 2, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    StreamMap<String, String> mapOperator = new StreamMap<>(new IdentityMap());
    streamConfig.setStreamOperator(mapOperator);
    streamConfig.setOperatorID(new OperatorID());
    StreamMockEnvironment environment = spy(testHarness.createEnvironment());
    // start the task
    testHarness.invoke(environment);
    testHarness.waitForTaskRunning();
    // emit cancellation barriers
    testHarness.processEvent(new CancelCheckpointMarker(2L), 0, 1);
    testHarness.processEvent(new CancelCheckpointMarker(2L), 0, 0);
    testHarness.waitForInputProcessing();
    // the decline call should go to the coordinator
    verify(environment, times(1)).declineCheckpoint(eq(2L), argThat(new CheckpointExceptionMatcher(CheckpointFailureReason.CHECKPOINT_DECLINED_ON_CANCELLATION_BARRIER)));
    // a cancellation barrier should be downstream
    Object result = testHarness.getOutput().poll();
    assertNotNull("nothing emitted", result);
    assertTrue("wrong type emitted", result instanceof CancelCheckpointMarker);
    assertEquals("wrong checkpoint id", 2L, ((CancelCheckpointMarker) result).getCheckpointId());
    // cancel and shutdown
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
}

Example 90 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StreamTaskSystemExitTest method createSystemExitTask.

private Task createSystemExitTask(final String invokableClassName, StreamOperator<?> operator) throws Exception {
    final Configuration taskConfiguration = new Configuration();
    final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
    streamConfig.setOperatorID(new OperatorID());
    streamConfig.setStreamOperator(operator);
    // for source run
    streamConfig.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    final JobInformation jobInformation = new JobInformation(new JobID(), "Test Job", new SerializedValue<>(new ExecutionConfig()), new Configuration(), Collections.emptyList(), Collections.emptyList());
    final TaskInformation taskInformation = new TaskInformation(new JobVertexID(), "Test Task", 1, 1, invokableClassName, taskConfiguration);
    final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();
    final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
    return new Task(jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0, 0, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), MemoryManagerBuilder.newBuilder().setMemorySize(32L * 1024L).build(), new IOManagerAsync(), shuffleEnvironment, new KvStateService(new KvStateRegistry(), null, null), mock(BroadcastVariableManager.class), new TaskEventDispatcher(), ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES, new TestTaskStateManager(), mock(TaskManagerActions.class), mock(InputSplitProvider.class), mock(CheckpointResponder.class), new NoOpTaskOperatorEventGateway(), new TestGlobalAggregateManager(), TestingClassLoaderLease.newBuilder().build(), mock(FileCache.class), taskManagerRuntimeInfo, UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(), new NoOpResultPartitionConsumableNotifier(), mock(PartitionProducerStateChecker.class), Executors.directExecutor());
}