Examples with OperatorID org.apache.flink.runtime.jobgraph.OperatorID used on opensource projects

Example 71 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class InterruptSensitiveRestoreTest method createTask.

// ------------------------------------------------------------------------
// Utilities
// ------------------------------------------------------------------------
private static Task createTask(StreamConfig streamConfig, Configuration taskConfig, StreamStateHandle state, int mode) throws IOException {
    ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
    Collection<KeyedStateHandle> keyedStateFromBackend = Collections.emptyList();
    Collection<KeyedStateHandle> keyedStateFromStream = Collections.emptyList();
    Collection<OperatorStateHandle> operatorStateBackend = Collections.emptyList();
    Collection<OperatorStateHandle> operatorStateStream = Collections.emptyList();
    Map<String, OperatorStateHandle.StateMetaInfo> operatorStateMetadata = new HashMap<>(1);
    OperatorStateHandle.StateMetaInfo metaInfo = new OperatorStateHandle.StateMetaInfo(new long[] { 0 }, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE);
    operatorStateMetadata.put(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME, metaInfo);
    KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(new KeyGroupRange(0, 0));
    Collection<OperatorStateHandle> operatorStateHandles = Collections.singletonList(new OperatorStreamStateHandle(operatorStateMetadata, state));
    List<KeyedStateHandle> keyedStateHandles = Collections.singletonList(new KeyGroupsStateHandle(keyGroupRangeOffsets, state));
    switch(mode) {
        case OPERATOR_MANAGED:
            operatorStateBackend = operatorStateHandles;
            break;
        case OPERATOR_RAW:
            operatorStateStream = operatorStateHandles;
            break;
        case KEYED_MANAGED:
            keyedStateFromBackend = keyedStateHandles;
            break;
        case KEYED_RAW:
            keyedStateFromStream = keyedStateHandles;
            break;
        default:
            throw new IllegalArgumentException();
    }
    OperatorSubtaskState operatorSubtaskState = OperatorSubtaskState.builder().setManagedOperatorState(new StateObjectCollection<>(operatorStateBackend)).setRawOperatorState(new StateObjectCollection<>(operatorStateStream)).setManagedKeyedState(new StateObjectCollection<>(keyedStateFromBackend)).setRawKeyedState(new StateObjectCollection<>(keyedStateFromStream)).build();
    JobVertexID jobVertexID = new JobVertexID();
    OperatorID operatorID = OperatorID.fromJobVertexID(jobVertexID);
    streamConfig.setOperatorID(operatorID);
    TaskStateSnapshot stateSnapshot = new TaskStateSnapshot();
    stateSnapshot.putSubtaskStateByOperatorID(operatorID, operatorSubtaskState);
    JobManagerTaskRestore taskRestore = new JobManagerTaskRestore(1L, stateSnapshot);
    JobInformation jobInformation = new JobInformation(new JobID(), "test job name", new SerializedValue<>(new ExecutionConfig()), new Configuration(), Collections.emptyList(), Collections.emptyList());
    TaskInformation taskInformation = new TaskInformation(jobVertexID, "test task name", 1, 1, SourceStreamTask.class.getName(), taskConfig);
    TestTaskStateManager taskStateManager = TestTaskStateManager.builder().setReportedCheckpointId(taskRestore.getRestoreCheckpointId()).setJobManagerTaskStateSnapshotsByCheckpointId(Collections.singletonMap(taskRestore.getRestoreCheckpointId(), taskRestore.getTaskStateSnapshot())).build();
    return new Task(jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0, 0, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), mock(MemoryManager.class), mock(IOManager.class), shuffleEnvironment, new KvStateService(new KvStateRegistry(), null, null), mock(BroadcastVariableManager.class), new TaskEventDispatcher(), ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES, taskStateManager, mock(TaskManagerActions.class), mock(InputSplitProvider.class), mock(CheckpointResponder.class), new NoOpTaskOperatorEventGateway(), new TestGlobalAggregateManager(), TestingClassLoaderLease.newBuilder().build(), new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() }, VoidPermanentBlobService.INSTANCE), new TestingTaskManagerRuntimeInfo(), UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(), new NoOpResultPartitionConsumableNotifier(), mock(PartitionProducerStateChecker.class), mock(Executor.class));
}

Example 72 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class SourceExternalCheckpointTriggerTest method testCheckpointsTriggeredBySource.

@Test
@SuppressWarnings("unchecked")
public void testCheckpointsTriggeredBySource() throws Exception {
    // set up the basic test harness
    final StreamTaskTestHarness<Long> testHarness = new StreamTaskTestHarness<>(SourceStreamTask::new, BasicTypeInfo.LONG_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    testHarness.getExecutionConfig().setLatencyTrackingInterval(-1);
    final long numElements = 10;
    final long checkpointEvery = 3;
    // set up the source function
    ExternalCheckpointsSource source = new ExternalCheckpointsSource(numElements, checkpointEvery);
    StreamConfig streamConfig = testHarness.getStreamConfig();
    StreamSource<Long, ?> sourceOperator = new StreamSource<>(source);
    streamConfig.setStreamOperator(sourceOperator);
    streamConfig.setOperatorID(new OperatorID());
    // this starts the source thread
    testHarness.invoke();
    final StreamTask<Long, ?> sourceTask = testHarness.getTask();
    ready.await();
    // now send an external trigger that should be ignored
    assertTrue(sourceTask.triggerCheckpointAsync(new CheckpointMetaData(32, 829), CheckpointOptions.forCheckpointWithDefaultLocation()).get());
    // step by step let the source thread emit elements
    sync.trigger();
    verifyNextElement(testHarness.getOutput(), 1L);
    sync.trigger();
    verifyNextElement(testHarness.getOutput(), 2L);
    sync.trigger();
    verifyNextElement(testHarness.getOutput(), 3L);
    verifyCheckpointBarrier(testHarness.getOutput(), 1L);
    sync.trigger();
    verifyNextElement(testHarness.getOutput(), 4L);
    // now send an regular trigger command that should be ignored
    assertTrue(sourceTask.triggerCheckpointAsync(new CheckpointMetaData(34, 900), CheckpointOptions.forCheckpointWithDefaultLocation()).get());
    sync.trigger();
    verifyNextElement(testHarness.getOutput(), 5L);
    sync.trigger();
    verifyNextElement(testHarness.getOutput(), 6L);
    verifyCheckpointBarrier(testHarness.getOutput(), 2L);
    for (long l = 7L, checkpoint = 3L; l <= numElements; l++) {
        sync.trigger();
        verifyNextElement(testHarness.getOutput(), l);
        if (l % checkpointEvery == 0) {
            verifyCheckpointBarrier(testHarness.getOutput(), checkpoint++);
        }
    }
// done!
}

Example 73 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class SourceStreamTaskTest method testClosingAllOperatorsOnChainProperly.

@Test
public void testClosingAllOperatorsOnChainProperly() throws Exception {
    final StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>(SourceStreamTask::new, STRING_TYPE_INFO);
    testHarness.setupOperatorChain(new OperatorID(), new OutputRecordInCloseTestSource<>("Source0", new FromElementsFunction<>(StringSerializer.INSTANCE, "Hello"))).chain(new OperatorID(), new TestBoundedOneInputStreamOperator("Operator1"), STRING_TYPE_INFO.createSerializer(new ExecutionConfig())).finish();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    streamConfig.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    testHarness.invoke();
    testHarness.waitForTaskCompletion();
    ArrayList<Object> expected = new ArrayList<>();
    Collections.addAll(expected, new StreamRecord<>("Hello"), new StreamRecord<>("[Source0]: End of input"), Watermark.MAX_WATERMARK, new StreamRecord<>("[Source0]: Finish"), new StreamRecord<>("[Operator1]: End of input"), new StreamRecord<>("[Operator1]: Finish"));
    final Object[] output = testHarness.getOutput().toArray();
    assertArrayEquals("Output was not correct.", expected.toArray(), output);
}

Example 74 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class SourceStreamTaskTest method testOpenClose.

/**
 * This test verifies that open() and close() are correctly called by the StreamTask.
 */
@Test
public void testOpenClose() throws Exception {
    final StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>(SourceStreamTask::new, STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    StreamSource<String, ?> sourceOperator = new StreamSource<>(new OpenCloseTestSource());
    streamConfig.setStreamOperator(sourceOperator);
    streamConfig.setOperatorID(new OperatorID());
    testHarness.invoke();
    testHarness.waitForTaskCompletion();
    assertTrue("RichFunction methods where not called.", OpenCloseTestSource.closeCalled);
    List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
    Assert.assertEquals(10, resultElements.size());
}

Example 75 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class OneInputStreamTaskTest method testCheckpointBarriers.

/**
 * This test verifies that checkpoint barriers are correctly forwarded.
 */
@Test
public void testCheckpointBarriers() throws Exception {
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, 2, 2, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    StreamMap<String, String> mapOperator = new StreamMap<>(new IdentityMap());
    streamConfig.setStreamOperator(mapOperator);
    streamConfig.setOperatorID(new OperatorID());
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    long initialTime = 0L;
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0);
    // These elements should be forwarded, since we did not yet receive a checkpoint barrier
    // on that input, only add to same input, otherwise we would not know the ordering
    // of the output since the Task might read the inputs in any order
    testHarness.processElement(new StreamRecord<>("Hello-1-1", initialTime), 1, 1);
    testHarness.processElement(new StreamRecord<>("Ciao-1-1", initialTime), 1, 1);
    expectedOutput.add(new StreamRecord<>("Hello-1-1", initialTime));
    expectedOutput.add(new StreamRecord<>("Ciao-1-1", initialTime));
    testHarness.waitForInputProcessing();
    // we should not yet see the barrier, only the two elements from non-blocked input
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1);
    testHarness.waitForInputProcessing();
    // now we should see the barrier
    expectedOutput.add(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()));
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}