Examples with OperatorID org.apache.flink.runtime.jobgraph.OperatorID used on opensource projects

Example 96 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class RestoreStreamTaskTest method testRestoreWithoutState.

@Test
public void testRestoreWithoutState() throws Exception {
    OperatorID headOperatorID = new OperatorID(42L, 42L);
    OperatorID tailOperatorID = new OperatorID(44L, 44L);
    JobManagerTaskRestore restore = createRunAndCheckpointOperatorChain(headOperatorID, new StatelessOperator(), tailOperatorID, new CounterOperator(), Optional.empty());
    TaskStateSnapshot stateHandles = restore.getTaskStateSnapshot();
    assertEquals(2, stateHandles.getSubtaskStateMappings().size());
    createRunAndCheckpointOperatorChain(headOperatorID, new StatelessOperator(), tailOperatorID, new CounterOperator(), Optional.of(restore));
    assertEquals(new HashSet<>(Arrays.asList(headOperatorID, tailOperatorID)), RESTORED_OPERATORS.keySet());
    assertThat(new HashSet<>(RESTORED_OPERATORS.values()), contains(restore.getRestoreCheckpointId()));
}

Example 97 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class RestoreStreamTaskTest method testRestoreHeadWithNewId.

@Test
public void testRestoreHeadWithNewId() throws Exception {
    OperatorID tailOperatorID = new OperatorID(44L, 44L);
    JobManagerTaskRestore restore = createRunAndCheckpointOperatorChain(new OperatorID(42L, 42L), new CounterOperator(), tailOperatorID, new CounterOperator(), Optional.empty());
    TaskStateSnapshot stateHandles = restore.getTaskStateSnapshot();
    assertEquals(2, stateHandles.getSubtaskStateMappings().size());
    createRunAndCheckpointOperatorChain(new OperatorID(4242L, 4242L), new CounterOperator(), tailOperatorID, new CounterOperator(), Optional.of(restore));
    assertEquals(Collections.singleton(tailOperatorID), RESTORED_OPERATORS.keySet());
    assertThat(new HashSet<>(RESTORED_OPERATORS.values()), contains(restore.getRestoreCheckpointId()));
}

Example 98 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class RestoreStreamTaskTest method testRestoreAfterScaleUp.

@Test
public void testRestoreAfterScaleUp() throws Exception {
    OperatorID headOperatorID = new OperatorID(42L, 42L);
    OperatorID tailOperatorID = new OperatorID(44L, 44L);
    JobManagerTaskRestore restore = createRunAndCheckpointOperatorChain(headOperatorID, new CounterOperator(), tailOperatorID, new CounterOperator(), Optional.empty());
    TaskStateSnapshot stateHandles = restore.getTaskStateSnapshot();
    assertEquals(2, stateHandles.getSubtaskStateMappings().size());
    // test empty state in case of scale up
    OperatorSubtaskState emptyHeadOperatorState = OperatorSubtaskState.builder().build();
    stateHandles.putSubtaskStateByOperatorID(headOperatorID, emptyHeadOperatorState);
    createRunAndCheckpointOperatorChain(headOperatorID, new CounterOperator(), tailOperatorID, new CounterOperator(), Optional.of(restore));
    assertEquals(new HashSet<>(Arrays.asList(headOperatorID, tailOperatorID)), RESTORED_OPERATORS.keySet());
    assertThat(new HashSet<>(RESTORED_OPERATORS.values()), contains(restore.getRestoreCheckpointId()));
}

Example 99 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class StreamTaskTerminationTest method testConcurrentAsyncCheckpointCannotFailFinishedStreamTask.

/**
 * FLINK-6833
 *
 * <p>Tests that a finished stream task cannot be failed by an asynchronous checkpointing
 * operation after the stream task has stopped running.
 */
@Test
public void testConcurrentAsyncCheckpointCannotFailFinishedStreamTask() throws Exception {
    final Configuration taskConfiguration = new Configuration();
    final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
    final NoOpStreamOperator<Long> noOpStreamOperator = new NoOpStreamOperator<>();
    final StateBackend blockingStateBackend = new BlockingStateBackend();
    streamConfig.setStreamOperator(noOpStreamOperator);
    streamConfig.setOperatorID(new OperatorID());
    streamConfig.setStateBackend(blockingStateBackend);
    final long checkpointId = 0L;
    final long checkpointTimestamp = 0L;
    final JobInformation jobInformation = new JobInformation(new JobID(), "Test Job", new SerializedValue<>(new ExecutionConfig()), new Configuration(), Collections.emptyList(), Collections.emptyList());
    final TaskInformation taskInformation = new TaskInformation(new JobVertexID(), "Test Task", 1, 1, BlockingStreamTask.class.getName(), taskConfiguration);
    final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();
    final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
    final Task task = new Task(jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0, 0, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), MemoryManagerBuilder.newBuilder().setMemorySize(32L * 1024L).build(), new IOManagerAsync(), shuffleEnvironment, new KvStateService(new KvStateRegistry(), null, null), mock(BroadcastVariableManager.class), new TaskEventDispatcher(), ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES, new TestTaskStateManager(), mock(TaskManagerActions.class), mock(InputSplitProvider.class), mock(CheckpointResponder.class), new NoOpTaskOperatorEventGateway(), new TestGlobalAggregateManager(), TestingClassLoaderLease.newBuilder().build(), mock(FileCache.class), taskManagerRuntimeInfo, UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(), new NoOpResultPartitionConsumableNotifier(), mock(PartitionProducerStateChecker.class), Executors.directExecutor());
    CompletableFuture<Void> taskRun = CompletableFuture.runAsync(() -> task.run(), TestingUtils.defaultExecutor());
    // wait until the stream task started running
    RUN_LATCH.await();
    // trigger a checkpoint
    task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, CheckpointOptions.forCheckpointWithDefaultLocation());
    // wait until the task has completed execution
    taskRun.get();
    // check that no failure occurred
    if (task.getFailureCause() != null) {
        throw new Exception("Task failed", task.getFailureCause());
    }
    // check that we have entered the finished state
    assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}

Example 100 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class TwoInputStreamTaskTest method testWatermarkMetrics.

@Test
@SuppressWarnings("unchecked")
public void testWatermarkMetrics() throws Exception {
    final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness = new TwoInputStreamTaskTestHarness<>(TwoInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    CoStreamMap<String, Integer, String> headOperator = new CoStreamMap<>(new IdentityMap());
    final OperatorID headOperatorId = new OperatorID();
    OneInputStreamTaskTest.WatermarkMetricOperator chainedOperator = new OneInputStreamTaskTest.WatermarkMetricOperator();
    OperatorID chainedOperatorId = new OperatorID();
    testHarness.setupOperatorChain(headOperatorId, headOperator).chain(chainedOperatorId, chainedOperator, BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig())).finish();
    InterceptingOperatorMetricGroup headOperatorMetricGroup = new InterceptingOperatorMetricGroup();
    InterceptingOperatorMetricGroup chainedOperatorMetricGroup = new InterceptingOperatorMetricGroup();
    InterceptingTaskMetricGroup taskMetricGroup = new InterceptingTaskMetricGroup() {

        @Override
        public InternalOperatorMetricGroup getOrAddOperator(OperatorID id, String name) {
            if (id.equals(headOperatorId)) {
                return headOperatorMetricGroup;
            } else if (id.equals(chainedOperatorId)) {
                return chainedOperatorMetricGroup;
            } else {
                return super.getOrAddOperator(id, name);
            }
        }
    };
    StreamMockEnvironment env = new StreamMockEnvironment(testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize, new TestTaskStateManager()) {

        @Override
        public TaskMetricGroup getMetricGroup() {
            return taskMetricGroup;
        }
    };
    testHarness.invoke(env);
    testHarness.waitForTaskRunning();
    Gauge<Long> taskInputWatermarkGauge = (Gauge<Long>) taskMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
    Gauge<Long> headInput1WatermarkGauge = (Gauge<Long>) headOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_1_WATERMARK);
    Gauge<Long> headInput2WatermarkGauge = (Gauge<Long>) headOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_2_WATERMARK);
    Gauge<Long> headInputWatermarkGauge = (Gauge<Long>) headOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
    Gauge<Long> headOutputWatermarkGauge = (Gauge<Long>) headOperatorMetricGroup.get(MetricNames.IO_CURRENT_OUTPUT_WATERMARK);
    Gauge<Long> chainedInputWatermarkGauge = (Gauge<Long>) chainedOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
    Gauge<Long> chainedOutputWatermarkGauge = (Gauge<Long>) chainedOperatorMetricGroup.get(MetricNames.IO_CURRENT_OUTPUT_WATERMARK);
    Assert.assertEquals("A metric was registered multiple times.", 7, new HashSet<>(Arrays.asList(taskInputWatermarkGauge, headInput1WatermarkGauge, headInput2WatermarkGauge, headInputWatermarkGauge, headOutputWatermarkGauge, chainedInputWatermarkGauge, chainedOutputWatermarkGauge)).size());
    Assert.assertEquals(Long.MIN_VALUE, taskInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headInput1WatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headInput2WatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headOutputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, chainedInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, chainedOutputWatermarkGauge.getValue().longValue());
    testHarness.processElement(new Watermark(1L), 0, 0);
    testHarness.waitForInputProcessing();
    Assert.assertEquals(Long.MIN_VALUE, taskInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(1L, headInput1WatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headInput2WatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headOutputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, chainedInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, chainedOutputWatermarkGauge.getValue().longValue());
    testHarness.processElement(new Watermark(2L), 1, 0);
    testHarness.waitForInputProcessing();
    Assert.assertEquals(1L, taskInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(1L, headInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(1L, headInput1WatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, headInput2WatermarkGauge.getValue().longValue());
    Assert.assertEquals(1L, headOutputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(1L, chainedInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, chainedOutputWatermarkGauge.getValue().longValue());
    testHarness.processElement(new Watermark(3L), 0, 0);
    testHarness.waitForInputProcessing();
    Assert.assertEquals(2L, taskInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, headInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(3L, headInput1WatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, headInput2WatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, headOutputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, chainedInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(4L, chainedOutputWatermarkGauge.getValue().longValue());
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
}