Examples with OperatorID org.apache.flink.runtime.jobgraph.OperatorID used on opensource projects

Example 86 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class InternalOperatorGroupTest method testCreateQueryServiceMetricInfo.

@Test
public void testCreateQueryServiceMetricInfo() {
    JobID jid = new JobID();
    JobVertexID vid = new JobVertexID();
    ExecutionAttemptID eid = new ExecutionAttemptID();
    OperatorID oid = new OperatorID();
    TaskManagerMetricGroup tm = TaskManagerMetricGroup.createTaskManagerMetricGroup(registry, "host", new ResourceID("id"));
    TaskMetricGroup task = tm.addJob(jid, "jobname").addTask(vid, eid, "taskName", 4, 5);
    InternalOperatorMetricGroup operator = task.getOrAddOperator(oid, "operator");
    QueryScopeInfo.OperatorQueryScopeInfo info = operator.createQueryServiceMetricInfo(new DummyCharacterFilter());
    assertEquals("", info.scope);
    assertEquals(jid.toString(), info.jobID);
    assertEquals(vid.toString(), info.vertexID);
    assertEquals(4, info.subtaskIndex);
    assertEquals("operator", info.operatorName);
}

Example 87 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class InternalOperatorGroupTest method testIOMetricGroupInstantiation.

@Test
public void testIOMetricGroupInstantiation() throws Exception {
    TaskManagerMetricGroup tmGroup = TaskManagerMetricGroup.createTaskManagerMetricGroup(registry, "theHostName", new ResourceID("test-tm-id"));
    TaskMetricGroup taskGroup = tmGroup.addJob(new JobID(), "myJobName").addTask(new JobVertexID(), new ExecutionAttemptID(), "aTaskName", 11, 0);
    InternalOperatorMetricGroup opGroup = taskGroup.getOrAddOperator(new OperatorID(), "myOpName");
    assertNotNull(opGroup.getIOMetricGroup());
    assertNotNull(opGroup.getIOMetricGroup().getNumRecordsInCounter());
    assertNotNull(opGroup.getIOMetricGroup().getNumRecordsOutCounter());
}

Example 88 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class InternalOperatorGroupTest method testVariables.

@Test
public void testVariables() {
    JobID jid = new JobID();
    JobVertexID tid = new JobVertexID();
    ExecutionAttemptID eid = new ExecutionAttemptID();
    OperatorID oid = new OperatorID();
    TaskManagerMetricGroup tmGroup = TaskManagerMetricGroup.createTaskManagerMetricGroup(registry, "theHostName", new ResourceID("test-tm-id"));
    TaskMetricGroup taskGroup = tmGroup.addJob(jid, "myJobName").addTask(tid, eid, "aTaskName", 11, 0);
    InternalOperatorMetricGroup opGroup = taskGroup.getOrAddOperator(oid, "myOpName");
    Map<String, String> variables = opGroup.getAllVariables();
    testVariable(variables, ScopeFormat.SCOPE_HOST, "theHostName");
    testVariable(variables, ScopeFormat.SCOPE_TASKMANAGER_ID, "test-tm-id");
    testVariable(variables, ScopeFormat.SCOPE_JOB_ID, jid.toString());
    testVariable(variables, ScopeFormat.SCOPE_JOB_NAME, "myJobName");
    testVariable(variables, ScopeFormat.SCOPE_TASK_VERTEX_ID, tid.toString());
    testVariable(variables, ScopeFormat.SCOPE_TASK_NAME, "aTaskName");
    testVariable(variables, ScopeFormat.SCOPE_TASK_ATTEMPT_ID, eid.toString());
    testVariable(variables, ScopeFormat.SCOPE_TASK_SUBTASK_INDEX, "11");
    testVariable(variables, ScopeFormat.SCOPE_TASK_ATTEMPT_NUM, "0");
    testVariable(variables, ScopeFormat.SCOPE_OPERATOR_ID, oid.toString());
    testVariable(variables, ScopeFormat.SCOPE_OPERATOR_NAME, "myOpName");
}

Example 89 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class SourceCoordinatorTest method testUserClassLoaderWhenCreatingNewEnumerator.

@Test
public void testUserClassLoaderWhenCreatingNewEnumerator() throws Exception {
    final ClassLoader testClassLoader = new URLClassLoader(new URL[0]);
    final OperatorCoordinator.Context context = new MockOperatorCoordinatorContext(new OperatorID(), testClassLoader);
    final EnumeratorCreatingSource<?, ClassLoaderTestEnumerator> source = new EnumeratorCreatingSource<>(ClassLoaderTestEnumerator::new);
    final SourceCoordinatorProvider<?> provider = new SourceCoordinatorProvider<>("testOperator", context.getOperatorId(), source, 1, WatermarkAlignmentParams.WATERMARK_ALIGNMENT_DISABLED);
    final OperatorCoordinator coordinator = provider.getCoordinator(context);
    coordinator.start();
    final ClassLoaderTestEnumerator enumerator = source.createEnumeratorFuture.get();
    assertSame(testClassLoader, enumerator.constructorClassLoader);
    assertSame(testClassLoader, enumerator.threadClassLoader.get());
    // cleanup
    coordinator.close();
}

Example 90 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class RocksDBAsyncSnapshotTest method testFullyAsyncSnapshot.

/**
 * This ensures that asynchronous state handles are actually materialized asynchronously.
 *
 * <p>We use latches to block at various stages and see if the code still continues through the
 * parts that are not asynchronous. If the checkpoint is not done asynchronously the test will
 * simply lock forever.
 */
@Test
public void testFullyAsyncSnapshot() throws Exception {
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    testHarness.configureForKeyedStream(new KeySelector<String, String>() {

        @Override
        public String getKey(String value) throws Exception {
            return value;
        }
    }, BasicTypeInfo.STRING_TYPE_INFO);
    StreamConfig streamConfig = testHarness.getStreamConfig();
    File dbDir = temporaryFolder.newFolder();
    RocksDBStateBackend backend = new RocksDBStateBackend(new MemoryStateBackend());
    backend.setDbStoragePath(dbDir.getAbsolutePath());
    streamConfig.setStateBackend(backend);
    streamConfig.setStreamOperator(new AsyncCheckpointOperator());
    streamConfig.setOperatorID(new OperatorID());
    final OneShotLatch delayCheckpointLatch = new OneShotLatch();
    final OneShotLatch ensureCheckpointLatch = new OneShotLatch();
    CheckpointResponder checkpointResponderMock = new CheckpointResponder() {

        @Override
        public void acknowledgeCheckpoint(JobID jobID, ExecutionAttemptID executionAttemptID, long checkpointId, CheckpointMetrics checkpointMetrics, TaskStateSnapshot subtaskState) {
            // even though the async checkpoint would not finish
            try {
                delayCheckpointLatch.await();
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
            boolean hasManagedKeyedState = false;
            for (Map.Entry<OperatorID, OperatorSubtaskState> entry : subtaskState.getSubtaskStateMappings()) {
                OperatorSubtaskState state = entry.getValue();
                if (state != null) {
                    hasManagedKeyedState |= state.getManagedKeyedState() != null;
                }
            }
            // should be one k/v state
            assertTrue(hasManagedKeyedState);
            // we now know that the checkpoint went through
            ensureCheckpointLatch.trigger();
        }

        @Override
        public void reportCheckpointMetrics(JobID jobID, ExecutionAttemptID executionAttemptID, long checkpointId, CheckpointMetrics checkpointMetrics) {
        }

        @Override
        public void declineCheckpoint(JobID jobID, ExecutionAttemptID executionAttemptID, long checkpointId, CheckpointException checkpointException) {
        }
    };
    JobID jobID = new JobID();
    ExecutionAttemptID executionAttemptID = new ExecutionAttemptID();
    TestTaskStateManager taskStateManagerTestMock = new TestTaskStateManager(jobID, executionAttemptID, checkpointResponderMock, TestLocalRecoveryConfig.disabled(), new InMemoryStateChangelogStorage(), new HashMap<>(), -1L, new OneShotLatch());
    StreamMockEnvironment mockEnv = new StreamMockEnvironment(testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize, taskStateManagerTestMock);
    AtomicReference<Throwable> errorRef = new AtomicReference<>();
    mockEnv.setExternalExceptionHandler(errorRef::set);
    testHarness.invoke(mockEnv);
    testHarness.waitForTaskRunning();
    final OneInputStreamTask<String, String> task = testHarness.getTask();
    task.triggerCheckpointAsync(new CheckpointMetaData(42, 17), CheckpointOptions.forCheckpointWithDefaultLocation()).get();
    testHarness.processElement(new StreamRecord<>("Wohoo", 0));
    // now we allow the checkpoint
    delayCheckpointLatch.trigger();
    // wait for the checkpoint to go through
    ensureCheckpointLatch.await();
    testHarness.endInput();
    ExecutorService threadPool = task.getAsyncOperationsThreadPool();
    threadPool.shutdown();
    Assert.assertTrue(threadPool.awaitTermination(60_000, TimeUnit.MILLISECONDS));
    testHarness.waitForTaskCompletion();
    if (errorRef.get() != null) {
        fail("Unexpected exception during execution.");
    }
}