Examples with StreamOperator org.apache.flink.streaming.api.operators.StreamOperator used on opensource projects

Example 11 with StreamOperator

use of org.apache.flink.streaming.api.operators.StreamOperator in project flink by apache.

the class StreamConfigChainer method finish.

public OWNER finish() {
    checkState(chainIndex > 0, "Use finishForSingletonOperatorChain");
    List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>();
    StreamNode sourceVertex = new StreamNode(chainIndex, null, null, (StreamOperator<?>) null, null, null);
    for (int i = 0; i < numberOfNonChainedOutputs; ++i) {
        StreamEdge streamEdge = new StreamEdge(sourceVertex, new StreamNode(chainIndex + i, null, null, (StreamOperator<?>) null, null, null), 0, new BroadcastPartitioner<>(), null);
        streamEdge.setBufferTimeout(1);
        outEdgesInOrder.add(streamEdge);
    }
    tailConfig.setChainEnd();
    tailConfig.setNumberOfOutputs(numberOfNonChainedOutputs);
    tailConfig.setOutEdgesInOrder(outEdgesInOrder);
    tailConfig.setNonChainedOutputs(outEdgesInOrder);
    headConfig.setTransitiveChainedTaskConfigs(chainedConfigs);
    headConfig.setOutEdgesInOrder(outEdgesInOrder);
    return owner;
}

Example 12 with StreamOperator

use of org.apache.flink.streaming.api.operators.StreamOperator in project flink by apache.

the class StreamTaskTest method testFailingCheckpointStreamOperator.

@Test
public void testFailingCheckpointStreamOperator() throws Exception {
    final long checkpointId = 42L;
    final long timestamp = 1L;
    TaskInfo mockTaskInfo = mock(TaskInfo.class);
    when(mockTaskInfo.getTaskNameWithSubtasks()).thenReturn("foobar");
    when(mockTaskInfo.getIndexOfThisSubtask()).thenReturn(0);
    Environment mockEnvironment = mock(Environment.class);
    when(mockEnvironment.getTaskInfo()).thenReturn(mockTaskInfo);
    StreamTask<?, AbstractStreamOperator<?>> streamTask = mock(StreamTask.class, Mockito.CALLS_REAL_METHODS);
    CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp);
    streamTask.setEnvironment(mockEnvironment);
    // mock the operators
    StreamOperator<?> streamOperator1 = mock(StreamOperator.class, withSettings().extraInterfaces(StreamCheckpointedOperator.class));
    StreamOperator<?> streamOperator2 = mock(StreamOperator.class, withSettings().extraInterfaces(StreamCheckpointedOperator.class));
    StreamOperator<?> streamOperator3 = mock(StreamOperator.class, withSettings().extraInterfaces(StreamCheckpointedOperator.class));
    // mock the returned snapshots
    OperatorSnapshotResult operatorSnapshotResult1 = mock(OperatorSnapshotResult.class);
    OperatorSnapshotResult operatorSnapshotResult2 = mock(OperatorSnapshotResult.class);
    final Exception testException = new Exception("Test exception");
    when(streamOperator1.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(operatorSnapshotResult1);
    when(streamOperator2.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(operatorSnapshotResult2);
    when(streamOperator3.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenThrow(testException);
    // mock the returned legacy snapshots
    StreamStateHandle streamStateHandle1 = mock(StreamStateHandle.class);
    StreamStateHandle streamStateHandle2 = mock(StreamStateHandle.class);
    StreamStateHandle streamStateHandle3 = mock(StreamStateHandle.class);
    when(streamOperator1.snapshotLegacyOperatorState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(streamStateHandle1);
    when(streamOperator2.snapshotLegacyOperatorState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(streamStateHandle2);
    when(streamOperator3.snapshotLegacyOperatorState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(streamStateHandle3);
    // set up the task
    StreamOperator<?>[] streamOperators = { streamOperator1, streamOperator2, streamOperator3 };
    OperatorChain<Void, AbstractStreamOperator<Void>> operatorChain = mock(OperatorChain.class);
    when(operatorChain.getAllOperators()).thenReturn(streamOperators);
    Whitebox.setInternalState(streamTask, "isRunning", true);
    Whitebox.setInternalState(streamTask, "lock", new Object());
    Whitebox.setInternalState(streamTask, "operatorChain", operatorChain);
    Whitebox.setInternalState(streamTask, "cancelables", new CloseableRegistry());
    Whitebox.setInternalState(streamTask, "configuration", new StreamConfig(new Configuration()));
    try {
        streamTask.triggerCheckpoint(checkpointMetaData, CheckpointOptions.forFullCheckpoint());
        fail("Expected test exception here.");
    } catch (Exception e) {
        assertEquals(testException, e.getCause());
    }
    verify(operatorSnapshotResult1).cancel();
    verify(operatorSnapshotResult2).cancel();
    verify(streamStateHandle1).discardState();
    verify(streamStateHandle2).discardState();
    verify(streamStateHandle3).discardState();
}

Example 13 with StreamOperator

use of org.apache.flink.streaming.api.operators.StreamOperator in project flink by apache.

the class StreamConfigChainer method chain.

public <IN, OUT> StreamConfigChainer<OWNER> chain(OperatorID operatorID, StreamOperatorFactory<OUT> operatorFactory, TypeSerializer<IN> inputSerializer, TypeSerializer<OUT> outputSerializer, boolean createKeyedStateBackend) {
    chainIndex++;
    StreamEdge streamEdge = new StreamEdge(new StreamNode(tailConfig.getChainIndex(), null, null, (StreamOperator<?>) null, null, null), new StreamNode(chainIndex, null, null, (StreamOperator<?>) null, null, null), 0, null, null);
    streamEdge.setBufferTimeout(bufferTimeout);
    tailConfig.setChainedOutputs(Collections.singletonList(streamEdge));
    tailConfig = new StreamConfig(new Configuration());
    tailConfig.setStreamOperatorFactory(checkNotNull(operatorFactory));
    tailConfig.setOperatorID(checkNotNull(operatorID));
    tailConfig.setupNetworkInputs(inputSerializer);
    tailConfig.setTypeSerializerOut(outputSerializer);
    if (createKeyedStateBackend) {
        // used to test multiple stateful operators chained in a single task.
        tailConfig.setStateKeySerializer(inputSerializer);
        tailConfig.setStateBackendUsesManagedMemory(true);
        tailConfig.setManagedMemoryFractionOperatorOfUseCase(ManagedMemoryUseCase.STATE_BACKEND, 1.0);
    }
    tailConfig.setChainIndex(chainIndex);
    chainedConfigs.put(chainIndex, tailConfig);
    return this;
}

Example 14 with StreamOperator

use of org.apache.flink.streaming.api.operators.StreamOperator in project flink by apache.

the class OneInputStreamTaskTest method configureChainedTestingStreamOperator.

// ==============================================================================================
// Utility functions and classes
// ==============================================================================================
private void configureChainedTestingStreamOperator(StreamConfig streamConfig, int numberChainedTasks) {
    Preconditions.checkArgument(numberChainedTasks >= 1, "The operator chain must at least " + "contain one operator.");
    TestingStreamOperator<Integer, Integer> previousOperator = new TestingStreamOperator<>();
    streamConfig.setStreamOperator(previousOperator);
    streamConfig.setOperatorID(new OperatorID(0L, 0L));
    // create the chain of operators
    Map<Integer, StreamConfig> chainedTaskConfigs = new HashMap<>(numberChainedTasks - 1);
    List<StreamEdge> outputEdges = new ArrayList<>(numberChainedTasks - 1);
    for (int chainedIndex = 1; chainedIndex < numberChainedTasks; chainedIndex++) {
        TestingStreamOperator<Integer, Integer> chainedOperator = new TestingStreamOperator<>();
        StreamConfig chainedConfig = new StreamConfig(new Configuration());
        chainedConfig.setupNetworkInputs(StringSerializer.INSTANCE);
        chainedConfig.setStreamOperator(chainedOperator);
        chainedConfig.setOperatorID(new OperatorID(0L, chainedIndex));
        chainedTaskConfigs.put(chainedIndex, chainedConfig);
        StreamEdge outputEdge = new StreamEdge(new StreamNode(chainedIndex - 1, null, null, (StreamOperator<?>) null, null, null), new StreamNode(chainedIndex, null, null, (StreamOperator<?>) null, null, null), 0, null, null);
        outputEdges.add(outputEdge);
    }
    streamConfig.setChainedOutputs(outputEdges);
    streamConfig.setTransitiveChainedTaskConfigs(chainedTaskConfigs);
}

Example 15 with StreamOperator

use of org.apache.flink.streaming.api.operators.StreamOperator in project flink by apache.

the class StreamTaskExecutionDecorationTest method before.

@Before
public void before() throws Exception {
    mailbox = new TaskMailboxImpl();
    decorator = new CountingStreamTaskActionExecutor();
    task = new StreamTask<Object, StreamOperator<Object>>(new DeclineDummyEnvironment(), null, FatalExitExceptionHandler.INSTANCE, decorator, mailbox) {

        @Override
        protected void init() {
        }

        @Override
        protected void processInput(MailboxDefaultAction.Controller controller) {
        }
    };
    task.operatorChain = new RegularOperatorChain<>(task, new NonRecordWriter<>());
}