Examples with ExecutionJobVertex org.apache.flink.runtime.executiongraph.ExecutionJobVertex used on opensource projects

Example 16 with ExecutionJobVertex

use of org.apache.flink.runtime.executiongraph.ExecutionJobVertex in project flink by apache.

the class CheckpointStatsTrackerTest method createTestTracker.

// ------------------------------------------------------------------------
/**
	 * Creates a "disabled" checkpoint tracker for tests.
	 */
static CheckpointStatsTracker createTestTracker() {
    ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
    when(jobVertex.getJobVertexId()).thenReturn(new JobVertexID());
    when(jobVertex.getParallelism()).thenReturn(1);
    return new CheckpointStatsTracker(0, Collections.singletonList(jobVertex), mock(JobSnapshottingSettings.class), new UnregisteredMetricsGroup());
}

Example 17 with ExecutionJobVertex

use of org.apache.flink.runtime.executiongraph.ExecutionJobVertex in project flink by apache.

the class KvStateLocationRegistryTest method testRegisterDuplicateName.

/**
	 * Tests that registrations with duplicate names throw an Exception.
	 */
@Test
public void testRegisterDuplicateName() throws Exception {
    ExecutionJobVertex[] vertices = new ExecutionJobVertex[] { createJobVertex(32), createJobVertex(13) };
    Map<JobVertexID, ExecutionJobVertex> vertexMap = createVertexMap(vertices);
    String registrationName = "duplicated-name";
    KvStateLocationRegistry registry = new KvStateLocationRegistry(new JobID(), vertexMap);
    // First operator registers
    registry.notifyKvStateRegistered(vertices[0].getJobVertexId(), new KeyGroupRange(0, 0), registrationName, new KvStateID(), new KvStateServerAddress(InetAddress.getLocalHost(), 12328));
    try {
        // Second operator registers same name
        registry.notifyKvStateRegistered(vertices[1].getJobVertexId(), new KeyGroupRange(0, 0), registrationName, new KvStateID(), new KvStateServerAddress(InetAddress.getLocalHost(), 12032));
        fail("Did not throw expected Exception after duplicated name");
    } catch (IllegalStateException ignored) {
    // Expected
    }
}

Example 18 with ExecutionJobVertex

use of org.apache.flink.runtime.executiongraph.ExecutionJobVertex in project flink by apache.

the class BackPressureStatsTrackerTest method testTriggerStackTraceSample.

/** Tests simple statistics with fake stack traces. */
@Test
@SuppressWarnings("unchecked")
public void testTriggerStackTraceSample() throws Exception {
    CompletableFuture<StackTraceSample> sampleFuture = new FlinkCompletableFuture<>();
    StackTraceSampleCoordinator sampleCoordinator = mock(StackTraceSampleCoordinator.class);
    when(sampleCoordinator.triggerStackTraceSample(any(ExecutionVertex[].class), anyInt(), any(Time.class), anyInt())).thenReturn(sampleFuture);
    ExecutionGraph graph = mock(ExecutionGraph.class);
    when(graph.getState()).thenReturn(JobStatus.RUNNING);
    // Same Thread execution context
    when(graph.getFutureExecutor()).thenReturn(new Executor() {

        @Override
        public void execute(Runnable runnable) {
            runnable.run();
        }
    });
    ExecutionVertex[] taskVertices = new ExecutionVertex[4];
    ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
    when(jobVertex.getJobId()).thenReturn(new JobID());
    when(jobVertex.getJobVertexId()).thenReturn(new JobVertexID());
    when(jobVertex.getGraph()).thenReturn(graph);
    when(jobVertex.getTaskVertices()).thenReturn(taskVertices);
    taskVertices[0] = mockExecutionVertex(jobVertex, 0);
    taskVertices[1] = mockExecutionVertex(jobVertex, 1);
    taskVertices[2] = mockExecutionVertex(jobVertex, 2);
    taskVertices[3] = mockExecutionVertex(jobVertex, 3);
    int numSamples = 100;
    Time delayBetweenSamples = Time.milliseconds(100L);
    BackPressureStatsTracker tracker = new BackPressureStatsTracker(sampleCoordinator, 9999, numSamples, delayBetweenSamples);
    // Trigger
    assertTrue("Failed to trigger", tracker.triggerStackTraceSample(jobVertex));
    verify(sampleCoordinator).triggerStackTraceSample(eq(taskVertices), eq(numSamples), eq(delayBetweenSamples), eq(BackPressureStatsTracker.MAX_STACK_TRACE_DEPTH));
    // Trigger again for pending request, should not fire
    assertFalse("Unexpected trigger", tracker.triggerStackTraceSample(jobVertex));
    assertTrue(tracker.getOperatorBackPressureStats(jobVertex).isEmpty());
    verify(sampleCoordinator).triggerStackTraceSample(eq(taskVertices), eq(numSamples), eq(delayBetweenSamples), eq(BackPressureStatsTracker.MAX_STACK_TRACE_DEPTH));
    assertTrue(tracker.getOperatorBackPressureStats(jobVertex).isEmpty());
    // Complete the future
    Map<ExecutionAttemptID, List<StackTraceElement[]>> traces = new HashMap<>();
    for (ExecutionVertex vertex : taskVertices) {
        List<StackTraceElement[]> taskTraces = new ArrayList<>();
        for (int i = 0; i < taskVertices.length; i++) {
            // Traces until sub task index are back pressured
            taskTraces.add(createStackTrace(i <= vertex.getParallelSubtaskIndex()));
        }
        traces.put(vertex.getCurrentExecutionAttempt().getAttemptId(), taskTraces);
    }
    int sampleId = 1231;
    int endTime = 841;
    StackTraceSample sample = new StackTraceSample(sampleId, 0, endTime, traces);
    // Succeed the promise
    sampleFuture.complete(sample);
    assertTrue(tracker.getOperatorBackPressureStats(jobVertex).isDefined());
    OperatorBackPressureStats stats = tracker.getOperatorBackPressureStats(jobVertex).get();
    // Verify the stats
    assertEquals(sampleId, stats.getSampleId());
    assertEquals(endTime, stats.getEndTimestamp());
    assertEquals(taskVertices.length, stats.getNumberOfSubTasks());
    for (int i = 0; i < taskVertices.length; i++) {
        double ratio = stats.getBackPressureRatio(i);
        // Traces until sub task index are back pressured
        assertEquals((i + 1) / ((double) 4), ratio, 0.0);
    }
}

Example 19 with ExecutionJobVertex

use of org.apache.flink.runtime.executiongraph.ExecutionJobVertex in project flink by apache.

the class StackTraceSampleCoordinatorITCase method testTaskClearedWhileSampling.

/**
	 * Tests that a cleared task is answered with a partial success response.
	 */
@Test
public void testTaskClearedWhileSampling() throws Exception {
    new JavaTestKit(testActorSystem) {

        {
            final FiniteDuration deadline = new FiniteDuration(60, TimeUnit.SECONDS);
            // The JobGraph
            final JobGraph jobGraph = new JobGraph();
            final int parallelism = 1;
            final JobVertex task = new JobVertex("Task");
            task.setInvokableClass(BlockingNoOpInvokable.class);
            task.setParallelism(parallelism);
            jobGraph.addVertex(task);
            ActorGateway jobManger = null;
            ActorGateway taskManager = null;
            try {
                jobManger = TestingUtils.createJobManager(testActorSystem, TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), new Configuration());
                final Configuration config = new Configuration();
                config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, parallelism);
                taskManager = TestingUtils.createTaskManager(testActorSystem, jobManger, config, true, true);
                final ActorGateway jm = jobManger;
                new Within(deadline) {

                    @Override
                    protected void run() {
                        try {
                            ActorGateway testActor = new AkkaActorGateway(getTestActor(), null);
                            int maxAttempts = 10;
                            int sleepTime = 100;
                            for (int i = 0; i < maxAttempts; i++, sleepTime *= 2) {
                                // Submit the job and wait until it is running
                                JobClient.submitJobDetached(jm, config, jobGraph, deadline, ClassLoader.getSystemClassLoader());
                                jm.tell(new WaitForAllVerticesToBeRunning(jobGraph.getJobID()), testActor);
                                expectMsgEquals(new AllVerticesRunning(jobGraph.getJobID()));
                                // Get the ExecutionGraph
                                jm.tell(new RequestExecutionGraph(jobGraph.getJobID()), testActor);
                                ExecutionGraphFound executionGraphResponse = expectMsgClass(ExecutionGraphFound.class);
                                ExecutionGraph executionGraph = (ExecutionGraph) executionGraphResponse.executionGraph();
                                ExecutionJobVertex vertex = executionGraph.getJobVertex(task.getID());
                                StackTraceSampleCoordinator coordinator = new StackTraceSampleCoordinator(testActorSystem.dispatcher(), 60000);
                                Future<StackTraceSample> sampleFuture = coordinator.triggerStackTraceSample(vertex.getTaskVertices(), // sampling.
                                21474700 * 100, Time.milliseconds(10L), 0);
                                // Wait before cancelling so that some samples
                                // are actually taken.
                                Thread.sleep(sleepTime);
                                // Cancel job
                                scala.concurrent.Future<?> removeFuture = jm.ask(new TestingJobManagerMessages.NotifyWhenJobRemoved(jobGraph.getJobID()), remaining());
                                jm.tell(new JobManagerMessages.CancelJob(jobGraph.getJobID()));
                                try {
                                    // Throws Exception on failure
                                    sampleFuture.get(remaining().toMillis(), TimeUnit.MILLISECONDS);
                                    // partial result.
                                    break;
                                } catch (Throwable t) {
                                // We were too fast in cancelling the job.
                                // Fall through and retry.
                                } finally {
                                    Await.ready(removeFuture, remaining());
                                }
                            }
                        } catch (Exception e) {
                            e.printStackTrace();
                            fail(e.getMessage());
                        }
                    }
                };
            } finally {
                TestingUtils.stopActor(jobManger);
                TestingUtils.stopActor(taskManager);
            }
        }
    };
}

Example 20 with ExecutionJobVertex

use of org.apache.flink.runtime.executiongraph.ExecutionJobVertex in project flink by apache.

the class JobVertexBackPressureHandlerTest method testResponseNoStatsAvailable.

/** Tests the response when no stats are available */
@Test
public void testResponseNoStatsAvailable() throws Exception {
    ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
    BackPressureStatsTracker statsTracker = mock(BackPressureStatsTracker.class);
    when(statsTracker.getOperatorBackPressureStats(any(ExecutionJobVertex.class))).thenReturn(Option.<OperatorBackPressureStats>empty());
    JobVertexBackPressureHandler handler = new JobVertexBackPressureHandler(mock(ExecutionGraphHolder.class), statsTracker, 9999);
    String response = handler.handleRequest(jobVertex, Collections.<String, String>emptyMap());
    ObjectMapper mapper = new ObjectMapper();
    JsonNode rootNode = mapper.readTree(response);
    // Single element
    assertEquals(1, rootNode.size());
    // Status
    JsonNode status = rootNode.get("status");
    assertNotNull(status);
    assertEquals("deprecated", status.textValue());
    verify(statsTracker).triggerStackTraceSample(any(ExecutionJobVertex.class));
}