Examples with StreamConfig org.apache.flink.streaming.api.graph.StreamConfig used on opensource projects

Example 36 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StreamTaskTest method testTaskAvoidHangingAfterSnapshotStateThrownException.

@Test
public void testTaskAvoidHangingAfterSnapshotStateThrownException() throws Exception {
    // given: Configured SourceStreamTask with source which fails on checkpoint.
    Configuration taskManagerConfig = new Configuration();
    taskManagerConfig.setString(STATE_BACKEND, TestMemoryStateBackendFactory.class.getName());
    StreamConfig cfg = new StreamConfig(new Configuration());
    cfg.setStateKeySerializer(mock(TypeSerializer.class));
    cfg.setOperatorID(new OperatorID(4712L, 43L));
    FailedSource failedSource = new FailedSource();
    cfg.setStreamOperator(new TestStreamSource<String, FailedSource>(failedSource));
    cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    try (NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build()) {
        Task task = createTask(SourceStreamTask.class, shuffleEnvironment, cfg, taskManagerConfig);
        // when: Task starts
        task.startTaskThread();
        // wait for the task starts doing the work.
        failedSource.awaitRunning();
        // and: Checkpoint is triggered which should lead to exception in Source.
        task.triggerCheckpointBarrier(42L, 1L, CheckpointOptions.forCheckpointWithDefaultLocation());
        // wait for clean termination.
        task.getExecutingThread().join();
        // then: The task doesn't hang but finished with FAILED state.
        assertEquals(ExecutionState.FAILED, task.getExecutionState());
    }
}

Example 37 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StreamTaskTest method testThreadInvariants.

/**
 * Tests that some StreamTask methods are called only in the main task's thread. Currently, the
 * main task's thread is the thread that creates the task.
 */
@Test
public void testThreadInvariants() throws Throwable {
    Configuration taskConfiguration = new Configuration();
    StreamConfig streamConfig = new StreamConfig(taskConfiguration);
    streamConfig.setStreamOperator(new StreamMap<>(value -> value));
    streamConfig.setOperatorID(new OperatorID());
    try (MockEnvironment mockEnvironment = new MockEnvironmentBuilder().setTaskConfiguration(taskConfiguration).build()) {
        ClassLoader taskClassLoader = new TestUserCodeClassLoader();
        RunningTask<ThreadInspectingTask> runningTask = runTask(() -> {
            Thread.currentThread().setContextClassLoader(taskClassLoader);
            return new ThreadInspectingTask(mockEnvironment);
        });
        runningTask.invocationFuture.get();
        assertThat(runningTask.streamTask.getTaskClassLoader(), is(sameInstance(taskClassLoader)));
    }
}

Example 38 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StreamTaskMailboxTestHarnessBuilder method initializeSourceInput.

private SourceInputConfig initializeSourceInput(int inputId, SourceInputConfigPlaceHolder sourceInput, StreamNode mainNode) {
    Map<Integer, StreamConfig> transitiveChainedTaskConfigs = streamConfig.getTransitiveChainedTaskConfigs(getClass().getClassLoader());
    Integer maxNodeId = transitiveChainedTaskConfigs.isEmpty() ? StreamConfigChainer.MAIN_NODE_ID : Collections.max(transitiveChainedTaskConfigs.keySet());
    List<StreamEdge> outEdgesInOrder = new LinkedList<>();
    StreamEdge sourceToMainEdge = new StreamEdge(new StreamNode(maxNodeId + inputId + 1337, null, null, (StreamOperator<?>) null, null, null), mainNode, 0, new ForwardPartitioner<>(), null);
    outEdgesInOrder.add(sourceToMainEdge);
    StreamConfig sourceConfig = new StreamConfig(new Configuration());
    sourceConfig.setTimeCharacteristic(streamConfig.getTimeCharacteristic());
    sourceConfig.setOutEdgesInOrder(outEdgesInOrder);
    sourceConfig.setChainedOutputs(outEdgesInOrder);
    sourceConfig.setTypeSerializerOut(sourceInput.getSourceSerializer());
    sourceConfig.setOperatorID(sourceInput.getOperatorId());
    sourceConfig.setStreamOperatorFactory(sourceInput.getSourceOperatorFactory());
    transitiveChainedTaskConfigs.put(sourceToMainEdge.getSourceId(), sourceConfig);
    streamConfig.setTransitiveChainedTaskConfigs(transitiveChainedTaskConfigs);
    return new SourceInputConfig(sourceToMainEdge);
}

Example 39 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StreamTaskTerminationTest method testConcurrentAsyncCheckpointCannotFailFinishedStreamTask.

/**
 * FLINK-6833
 *
 * <p>Tests that a finished stream task cannot be failed by an asynchronous checkpointing
 * operation after the stream task has stopped running.
 */
@Test
public void testConcurrentAsyncCheckpointCannotFailFinishedStreamTask() throws Exception {
    final Configuration taskConfiguration = new Configuration();
    final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
    final NoOpStreamOperator<Long> noOpStreamOperator = new NoOpStreamOperator<>();
    final StateBackend blockingStateBackend = new BlockingStateBackend();
    streamConfig.setStreamOperator(noOpStreamOperator);
    streamConfig.setOperatorID(new OperatorID());
    streamConfig.setStateBackend(blockingStateBackend);
    final long checkpointId = 0L;
    final long checkpointTimestamp = 0L;
    final JobInformation jobInformation = new JobInformation(new JobID(), "Test Job", new SerializedValue<>(new ExecutionConfig()), new Configuration(), Collections.emptyList(), Collections.emptyList());
    final TaskInformation taskInformation = new TaskInformation(new JobVertexID(), "Test Task", 1, 1, BlockingStreamTask.class.getName(), taskConfiguration);
    final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();
    final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
    final Task task = new Task(jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0, 0, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), MemoryManagerBuilder.newBuilder().setMemorySize(32L * 1024L).build(), new IOManagerAsync(), shuffleEnvironment, new KvStateService(new KvStateRegistry(), null, null), mock(BroadcastVariableManager.class), new TaskEventDispatcher(), ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES, new TestTaskStateManager(), mock(TaskManagerActions.class), mock(InputSplitProvider.class), mock(CheckpointResponder.class), new NoOpTaskOperatorEventGateway(), new TestGlobalAggregateManager(), TestingClassLoaderLease.newBuilder().build(), mock(FileCache.class), taskManagerRuntimeInfo, UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(), new NoOpResultPartitionConsumableNotifier(), mock(PartitionProducerStateChecker.class), Executors.directExecutor());
    CompletableFuture<Void> taskRun = CompletableFuture.runAsync(() -> task.run(), TestingUtils.defaultExecutor());
    // wait until the stream task started running
    RUN_LATCH.await();
    // trigger a checkpoint
    task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, CheckpointOptions.forCheckpointWithDefaultLocation());
    // wait until the task has completed execution
    taskRun.get();
    // check that no failure occurred
    if (task.getFailureCause() != null) {
        throw new Exception("Task failed", task.getFailureCause());
    }
    // check that we have entered the finished state
    assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}

Example 40 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StreamTaskCancellationTest method testCancelTaskExceptionHandling.

/**
 * CancelTaskException can be thrown in a down stream task, for example if an upstream task was
 * cancelled first and those two tasks were connected via {@link
 * org.apache.flink.runtime.io.network.partition.consumer.LocalInputChannel}. {@link StreamTask}
 * should be able to correctly handle such situation.
 */
@Test
public void testCancelTaskExceptionHandling() throws Exception {
    StreamConfig cfg = new StreamConfig(new Configuration());
    try (NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build()) {
        Task task = createTask(CancelThrowingTask.class, shuffleEnvironment, cfg, new Configuration());
        task.startTaskThread();
        task.getExecutingThread().join();
        assertEquals(ExecutionState.CANCELED, task.getExecutionState());
    }
}