Examples with OperatorID org.apache.flink.runtime.jobgraph.OperatorID used on opensource projects

Example 66 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class MultipleInputStreamTaskTest method testSkipExecutionsIfFinishedOnRestore.

@Test
public void testSkipExecutionsIfFinishedOnRestore() throws Exception {
    OperatorID nonSourceOperatorId = new OperatorID();
    try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).setCollectNetworkEvents().modifyStreamConfig(config -> config.setCheckpointingEnabled(true)).modifyExecutionConfig(applyObjectReuse(objectReuse)).addInput(BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.INT_TYPE_INFO).setTaskStateSnapshot(1, TaskStateSnapshot.FINISHED_ON_RESTORE).setupOperatorChain(nonSourceOperatorId, new LifeCycleMonitorMultipleInputOperatorFactory()).chain(new TestFinishedOnRestoreStreamOperator(), StringSerializer.INSTANCE).finish().build()) {
        testHarness.processElement(Watermark.MAX_WATERMARK, 0);
        testHarness.processElement(Watermark.MAX_WATERMARK, 1);
        testHarness.processElement(Watermark.MAX_WATERMARK, 2);
        testHarness.waitForTaskCompletion();
        assertThat(testHarness.getOutput(), contains(Watermark.MAX_WATERMARK, new EndOfData(StopMode.DRAIN)));
    }
}

Example 67 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class MultipleInputStreamTaskTest method testMetrics.

/**
 * With chained sources, task's and main operator's number of input records are two different
 * things. The first one should take into account only records comming in from the network,
 * ignoring records produced inside the task itself (like via a chained source). Main operator
 * should on the other hand report all records from all of the inputs (regardless if it's a
 * network or chained input).
 */
@Test
public void testMetrics() throws Exception {
    HashMap<String, OperatorMetricGroup> operatorMetrics = new HashMap<>();
    TaskMetricGroup taskMetricGroup = new UnregisteredMetricGroups.UnregisteredTaskMetricGroup() {

        @Override
        public InternalOperatorMetricGroup getOrAddOperator(OperatorID operatorID, String name) {
            InternalOperatorMetricGroup operatorMetricGroup = super.getOrAddOperator(operatorID, name);
            operatorMetrics.put(name, operatorMetricGroup);
            return operatorMetricGroup;
        }
    };
    String mainOperatorName = "MainOperator";
    try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).modifyExecutionConfig(applyObjectReuse(objectReuse)).addInput(BasicTypeInfo.STRING_TYPE_INFO).addSourceInput(new SourceOperatorFactory<>(new LifeCycleTrackingMockSource(Boundedness.BOUNDED, 1), WatermarkStrategy.noWatermarks()), BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.STRING_TYPE_INFO).setupOperatorChain(new MapToStringMultipleInputOperatorFactory(3)).name(mainOperatorName).chain(new OneInputStreamTaskTest.DuplicatingOperator(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig())).chain(new OneInputStreamTaskTest.DuplicatingOperator(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig())).chain(new OneInputStreamTaskTest.DuplicatingOperator(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig())).finish().setTaskMetricGroup(taskMetricGroup).build()) {
        assertTrue(operatorMetrics.containsKey(mainOperatorName));
        OperatorMetricGroup mainOperatorMetrics = operatorMetrics.get(mainOperatorName);
        Counter numRecordsInCounter = taskMetricGroup.getIOMetricGroup().getNumRecordsInCounter();
        Counter numRecordsOutCounter = taskMetricGroup.getIOMetricGroup().getNumRecordsOutCounter();
        int numRecords1 = 5;
        int numRecords2 = 3;
        int numRecords3 = 2;
        // end prematurely
        for (int x = 0; x < numRecords2; x++) {
            addSourceRecords(testHarness, 1, 42);
        }
        for (int x = 0; x < numRecords1; x++) {
            testHarness.processElement(new StreamRecord<>("hello"), 0, 0);
        }
        for (int x = 0; x < numRecords3; x++) {
            testHarness.processElement(new StreamRecord<>("hello"), 1, 0);
        }
        int networkRecordsIn = numRecords1 + numRecords3;
        int mainOperatorRecordsIn = networkRecordsIn + numRecords2;
        int totalRecordsOut = mainOperatorRecordsIn * 2 * 2 * // there are three operators duplicating the records
        2;
        assertEquals(mainOperatorRecordsIn, mainOperatorMetrics.getIOMetricGroup().getNumRecordsInCounter().getCount());
        assertEquals(networkRecordsIn, numRecordsInCounter.getCount());
        assertEquals(totalRecordsOut, numRecordsOutCounter.getCount());
        testHarness.waitForTaskCompletion();
    }
}

Example 68 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class LocalStateForwardingTest method testReportingFromSnapshotToTaskStateManager.

/**
 * This tests the forwarding of jm and tm-local state from the futures reported by the backends,
 * through the async checkpointing thread to the {@link
 * org.apache.flink.runtime.state.TaskStateManager}.
 */
@Test
public void testReportingFromSnapshotToTaskStateManager() throws Exception {
    TestTaskStateManager taskStateManager = new TestTaskStateManager();
    StreamMockEnvironment streamMockEnvironment = new StreamMockEnvironment(new Configuration(), new Configuration(), new ExecutionConfig(), 1024 * 1024, new MockInputSplitProvider(), 0, taskStateManager);
    StreamTask testStreamTask = new StreamTaskTest.NoOpStreamTask(streamMockEnvironment);
    CheckpointMetaData checkpointMetaData = new CheckpointMetaData(0L, 0L);
    CheckpointMetricsBuilder checkpointMetrics = new CheckpointMetricsBuilder();
    Map<OperatorID, OperatorSnapshotFutures> snapshots = new HashMap<>(1);
    OperatorSnapshotFutures osFuture = new OperatorSnapshotFutures();
    osFuture.setKeyedStateManagedFuture(createSnapshotResult(KeyedStateHandle.class));
    osFuture.setKeyedStateRawFuture(createSnapshotResult(KeyedStateHandle.class));
    osFuture.setOperatorStateManagedFuture(createSnapshotResult(OperatorStateHandle.class));
    osFuture.setOperatorStateRawFuture(createSnapshotResult(OperatorStateHandle.class));
    osFuture.setInputChannelStateFuture(createSnapshotCollectionResult(InputChannelStateHandle.class));
    osFuture.setResultSubpartitionStateFuture(createSnapshotCollectionResult(ResultSubpartitionStateHandle.class));
    OperatorID operatorID = new OperatorID();
    snapshots.put(operatorID, osFuture);
    AsyncCheckpointRunnable checkpointRunnable = new AsyncCheckpointRunnable(snapshots, checkpointMetaData, checkpointMetrics, 0L, testStreamTask.getName(), asyncCheckpointRunnable -> {
    }, testStreamTask.getEnvironment(), testStreamTask, false, false, () -> true);
    checkpointMetrics.setAlignmentDurationNanos(0L);
    checkpointMetrics.setBytesProcessedDuringAlignment(0L);
    checkpointRunnable.run();
    TaskStateSnapshot lastJobManagerTaskStateSnapshot = taskStateManager.getLastJobManagerTaskStateSnapshot();
    TaskStateSnapshot lastTaskManagerTaskStateSnapshot = taskStateManager.getLastTaskManagerTaskStateSnapshot();
    OperatorSubtaskState jmState = lastJobManagerTaskStateSnapshot.getSubtaskStateByOperatorID(operatorID);
    OperatorSubtaskState tmState = lastTaskManagerTaskStateSnapshot.getSubtaskStateByOperatorID(operatorID);
    performCheck(osFuture.getKeyedStateManagedFuture(), jmState.getManagedKeyedState(), tmState.getManagedKeyedState());
    performCheck(osFuture.getKeyedStateRawFuture(), jmState.getRawKeyedState(), tmState.getRawKeyedState());
    performCheck(osFuture.getOperatorStateManagedFuture(), jmState.getManagedOperatorState(), tmState.getManagedOperatorState());
    performCheck(osFuture.getOperatorStateRawFuture(), jmState.getRawOperatorState(), tmState.getRawOperatorState());
    performCollectionCheck(osFuture.getInputChannelStateFuture(), jmState.getInputChannelState(), tmState.getInputChannelState());
    performCollectionCheck(osFuture.getResultSubpartitionStateFuture(), jmState.getResultSubpartitionState(), tmState.getResultSubpartitionState());
}

Example 69 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class OneInputStreamTaskTest method testWatermarkAndWatermarkStatusForwarding.

/**
 * This test verifies that watermarks and watermark statuses are correctly forwarded. This also
 * checks whether watermarks are forwarded only when we have received watermarks from all
 * inputs. The forwarded watermark must be the minimum of the watermarks of all active inputs.
 */
@Test
public void testWatermarkAndWatermarkStatusForwarding() throws Exception {
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, 2, 2, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    StreamMap<String, String> mapOperator = new StreamMap<>(new IdentityMap());
    streamConfig.setStreamOperator(mapOperator);
    streamConfig.setOperatorID(new OperatorID());
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    long initialTime = 0L;
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    testHarness.processElement(new Watermark(initialTime), 0, 0);
    testHarness.processElement(new Watermark(initialTime), 0, 1);
    testHarness.processElement(new Watermark(initialTime), 1, 0);
    // now the output should still be empty
    testHarness.waitForInputProcessing();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processElement(new Watermark(initialTime), 1, 1);
    // now the watermark should have propagated, Map simply forward Watermarks
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // contrary to checkpoint barriers these elements are not blocked by watermarks
    testHarness.processElement(new StreamRecord<>("Hello", initialTime));
    testHarness.processElement(new StreamRecord<>("Ciao", initialTime));
    expectedOutput.add(new StreamRecord<>("Hello", initialTime));
    expectedOutput.add(new StreamRecord<>("Ciao", initialTime));
    testHarness.processElement(new Watermark(initialTime + 4), 0, 0);
    testHarness.processElement(new Watermark(initialTime + 3), 0, 1);
    testHarness.processElement(new Watermark(initialTime + 3), 1, 0);
    testHarness.processElement(new Watermark(initialTime + 2), 1, 1);
    // check whether we get the minimum of all the watermarks, this must also only occur in
    // the output after the two StreamRecords
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 2));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // advance watermark from one of the inputs, now we should get a new one since the
    // minimum increases
    testHarness.processElement(new Watermark(initialTime + 4), 1, 1);
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 3));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // advance the other two inputs, now we should get a new one since the
    // minimum increases again
    testHarness.processElement(new Watermark(initialTime + 4), 0, 1);
    testHarness.processElement(new Watermark(initialTime + 4), 1, 0);
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 4));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // test whether idle input channels are acknowledged correctly when forwarding watermarks
    testHarness.processElement(WatermarkStatus.IDLE, 0, 1);
    testHarness.processElement(WatermarkStatus.IDLE, 1, 0);
    testHarness.processElement(new Watermark(initialTime + 6), 0, 0);
    testHarness.processElement(new Watermark(initialTime + 5), 1, // this watermark should be advanced first
    1);
    // once this is acknowledged,
    testHarness.processElement(WatermarkStatus.IDLE, 1, 1);
    // watermark (initial + 6) should be forwarded
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 5));
    expectedOutput.add(new Watermark(initialTime + 6));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // make all input channels idle and check that the operator's idle status is forwarded
    testHarness.processElement(WatermarkStatus.IDLE, 0, 0);
    testHarness.waitForInputProcessing();
    expectedOutput.add(WatermarkStatus.IDLE);
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // make some input channels active again and check that the operator's active status is
    // forwarded only once
    testHarness.processElement(WatermarkStatus.ACTIVE, 1, 0);
    testHarness.processElement(WatermarkStatus.ACTIVE, 0, 1);
    testHarness.waitForInputProcessing();
    expectedOutput.add(WatermarkStatus.ACTIVE);
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
    assertEquals(2, resultElements.size());
}

Example 70 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class OneInputStreamTaskTest method testQuiesceTimerServiceAfterOpClose.

@Test
public void testQuiesceTimerServiceAfterOpClose() throws Exception {
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, 2, 2, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    streamConfig.setStreamOperator(new TestOperator());
    streamConfig.setOperatorID(new OperatorID());
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    SystemProcessingTimeService timeService = (SystemProcessingTimeService) testHarness.getTimerService();
    // verify that the timer service is running
    Assert.assertTrue(timeService.isAlive());
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    timeService.shutdownService();
}