Examples with ExecutionJobVertex org.apache.flink.runtime.executiongraph.ExecutionJobVertex used on opensource projects

Example 21 with ExecutionJobVertex

use of org.apache.flink.runtime.executiongraph.ExecutionJobVertex in project flink by apache.

the class JobVertexBackPressureHandlerTest method testResponsePassedRefreshInterval.

/** Tests that after the refresh interval another sample is triggered. */
@Test
public void testResponsePassedRefreshInterval() throws Exception {
    ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
    BackPressureStatsTracker statsTracker = mock(BackPressureStatsTracker.class);
    OperatorBackPressureStats stats = new OperatorBackPressureStats(0, System.currentTimeMillis(), new double[] { 0.31, 0.48, 1.0, 0.0 });
    when(statsTracker.getOperatorBackPressureStats(any(ExecutionJobVertex.class))).thenReturn(Option.apply(stats));
    JobVertexBackPressureHandler handler = new JobVertexBackPressureHandler(mock(ExecutionGraphHolder.class), statsTracker, // <----- refresh interval should fire immediately
    0);
    String response = handler.handleRequest(jobVertex, Collections.<String, String>emptyMap());
    ObjectMapper mapper = new ObjectMapper();
    JsonNode rootNode = mapper.readTree(response);
    // Single element
    assertEquals(4, rootNode.size());
    // Status
    JsonNode status = rootNode.get("status");
    assertNotNull(status);
    // Interval passed, hence deprecated
    assertEquals("deprecated", status.textValue());
    // Back pressure level
    JsonNode backPressureLevel = rootNode.get("backpressure-level");
    assertNotNull(backPressureLevel);
    assertEquals("high", backPressureLevel.textValue());
    // End time stamp
    JsonNode endTimeStamp = rootNode.get("end-timestamp");
    assertNotNull(endTimeStamp);
    assertEquals(stats.getEndTimestamp(), endTimeStamp.longValue());
    // Subtasks
    JsonNode subTasks = rootNode.get("subtasks");
    assertEquals(stats.getNumberOfSubTasks(), subTasks.size());
    for (int i = 0; i < subTasks.size(); i++) {
        JsonNode subTask = subTasks.get(i);
        JsonNode index = subTask.get("subtask");
        assertEquals(i, index.intValue());
        JsonNode level = subTask.get("backpressure-level");
        assertEquals(JobVertexBackPressureHandler.getBackPressureLevel(stats.getBackPressureRatio(i)), level.textValue());
        JsonNode ratio = subTask.get("ratio");
        assertEquals(stats.getBackPressureRatio(i), ratio.doubleValue(), 0.0);
    }
    // Verify triggered
    verify(statsTracker).triggerStackTraceSample(any(ExecutionJobVertex.class));
}

Example 22 with ExecutionJobVertex

use of org.apache.flink.runtime.executiongraph.ExecutionJobVertex in project flink by apache.

the class CheckpointStatsTracker method createEmptyTaskStateStatsMap.

/**
	 * Creates an empty map with a {@link TaskStateStats} instance per task
	 * that is involved in the checkpoint.
	 *
	 * @return An empty map with an {@link TaskStateStats} entry for each task that is involved in the checkpoint.
	 */
private ConcurrentHashMap<JobVertexID, TaskStateStats> createEmptyTaskStateStatsMap() {
    ConcurrentHashMap<JobVertexID, TaskStateStats> taskStatsMap = new ConcurrentHashMap<>(jobVertices.size());
    for (ExecutionJobVertex vertex : jobVertices) {
        TaskStateStats taskStats = new TaskStateStats(vertex.getJobVertexId(), vertex.getParallelism());
        taskStatsMap.put(vertex.getJobVertexId(), taskStats);
    }
    return taskStatsMap;
}

Example 23 with ExecutionJobVertex

use of org.apache.flink.runtime.executiongraph.ExecutionJobVertex in project flink by apache.

the class StateAssignmentOperation method assignStates.

public boolean assignStates() throws Exception {
    // this tracks if we find missing node hash ids and already use secondary mappings
    boolean expandedToLegacyIds = false;
    Map<JobVertexID, ExecutionJobVertex> localTasks = this.tasks;
    for (Map.Entry<JobVertexID, TaskState> taskGroupStateEntry : taskStates.entrySet()) {
        TaskState taskState = taskGroupStateEntry.getValue();
        //----------------------------------------find vertex for state---------------------------------------------
        ExecutionJobVertex executionJobVertex = localTasks.get(taskGroupStateEntry.getKey());
        // for example as generated from older flink versions, to provide backwards compatibility.
        if (executionJobVertex == null && !expandedToLegacyIds) {
            localTasks = ExecutionJobVertex.includeLegacyJobVertexIDs(localTasks);
            executionJobVertex = localTasks.get(taskGroupStateEntry.getKey());
            expandedToLegacyIds = true;
            logger.info("Could not find ExecutionJobVertex. Including legacy JobVertexIDs in search.");
        }
        if (executionJobVertex == null) {
            if (allowNonRestoredState) {
                logger.info("Skipped checkpoint state for operator {}.", taskState.getJobVertexID());
                continue;
            } else {
                throw new IllegalStateException("There is no execution job vertex for the job" + " vertex ID " + taskGroupStateEntry.getKey());
            }
        }
        checkParallelismPreconditions(taskState, executionJobVertex);
        assignTaskStatesToOperatorInstances(taskState, executionJobVertex);
    }
    return true;
}

Example 24 with ExecutionJobVertex

use of org.apache.flink.runtime.executiongraph.ExecutionJobVertex in project flink by apache.

the class KvStateLocationRegistry method notifyKvStateRegistered.

/**
	 * Notifies the registry about a registered KvState instance.
	 *
	 * @param jobVertexId JobVertexID the KvState instance belongs to
	 * @param keyGroupRange Key group range the KvState instance belongs to
	 * @param registrationName Name under which the KvState has been registered
	 * @param kvStateId ID of the registered KvState instance
	 * @param kvStateServerAddress Server address where to find the KvState instance
	 *
	 * @throws IllegalArgumentException If JobVertexID does not belong to job
	 * @throws IllegalArgumentException If state has been registered with same
	 * name by another operator.
	 * @throws IndexOutOfBoundsException If key group index is out of bounds.
	 */
public void notifyKvStateRegistered(JobVertexID jobVertexId, KeyGroupRange keyGroupRange, String registrationName, KvStateID kvStateId, KvStateServerAddress kvStateServerAddress) {
    KvStateLocation location = lookupTable.get(registrationName);
    if (location == null) {
        // First registration for this operator, create the location info
        ExecutionJobVertex vertex = jobVertices.get(jobVertexId);
        if (vertex != null) {
            int parallelism = vertex.getMaxParallelism();
            location = new KvStateLocation(jobId, jobVertexId, parallelism, registrationName);
            lookupTable.put(registrationName, location);
        } else {
            throw new IllegalArgumentException("Unknown JobVertexID " + jobVertexId);
        }
    }
    // Duplicated name if vertex IDs don't match
    if (!location.getJobVertexId().equals(jobVertexId)) {
        IllegalStateException duplicate = new IllegalStateException("Registration name clash. KvState with name '" + registrationName + "' has already been registered by another operator (" + location.getJobVertexId() + ").");
        ExecutionJobVertex vertex = jobVertices.get(jobVertexId);
        if (vertex != null) {
            vertex.fail(new SuppressRestartsException(duplicate));
        }
        throw duplicate;
    }
    location.registerKvState(keyGroupRange, kvStateId, kvStateServerAddress);
}

Example 25 with ExecutionJobVertex

use of org.apache.flink.runtime.executiongraph.ExecutionJobVertex in project flink by apache.

the class CheckpointStateRestoreTest method testSetState.

/**
	 * Tests that on restore the task state is reset for each stateful task.
	 */
@Test
public void testSetState() {
    try {
        final ChainedStateHandle<StreamStateHandle> serializedState = CheckpointCoordinatorTest.generateChainedStateHandle(new SerializableObject());
        KeyGroupRange keyGroupRange = KeyGroupRange.of(0, 0);
        List<SerializableObject> testStates = Collections.singletonList(new SerializableObject());
        final KeyGroupsStateHandle serializedKeyGroupStates = CheckpointCoordinatorTest.generateKeyGroupState(keyGroupRange, testStates);
        final JobID jid = new JobID();
        final JobVertexID statefulId = new JobVertexID();
        final JobVertexID statelessId = new JobVertexID();
        Execution statefulExec1 = mockExecution();
        Execution statefulExec2 = mockExecution();
        Execution statefulExec3 = mockExecution();
        Execution statelessExec1 = mockExecution();
        Execution statelessExec2 = mockExecution();
        ExecutionVertex stateful1 = mockExecutionVertex(statefulExec1, statefulId, 0, 3);
        ExecutionVertex stateful2 = mockExecutionVertex(statefulExec2, statefulId, 1, 3);
        ExecutionVertex stateful3 = mockExecutionVertex(statefulExec3, statefulId, 2, 3);
        ExecutionVertex stateless1 = mockExecutionVertex(statelessExec1, statelessId, 0, 2);
        ExecutionVertex stateless2 = mockExecutionVertex(statelessExec2, statelessId, 1, 2);
        ExecutionJobVertex stateful = mockExecutionJobVertex(statefulId, new ExecutionVertex[] { stateful1, stateful2, stateful3 });
        ExecutionJobVertex stateless = mockExecutionJobVertex(statelessId, new ExecutionVertex[] { stateless1, stateless2 });
        Map<JobVertexID, ExecutionJobVertex> map = new HashMap<JobVertexID, ExecutionJobVertex>();
        map.put(statefulId, stateful);
        map.put(statelessId, stateless);
        CheckpointCoordinator coord = new CheckpointCoordinator(jid, 200000L, 200000L, 0, Integer.MAX_VALUE, ExternalizedCheckpointSettings.none(), new ExecutionVertex[] { stateful1, stateful2, stateful3, stateless1, stateless2 }, new ExecutionVertex[] { stateful1, stateful2, stateful3, stateless1, stateless2 }, new ExecutionVertex[0], new StandaloneCheckpointIDCounter(), new StandaloneCompletedCheckpointStore(1), null, Executors.directExecutor());
        // create ourselves a checkpoint with state
        final long timestamp = 34623786L;
        coord.triggerCheckpoint(timestamp, false);
        PendingCheckpoint pending = coord.getPendingCheckpoints().values().iterator().next();
        final long checkpointId = pending.getCheckpointId();
        SubtaskState checkpointStateHandles = new SubtaskState(serializedState, null, null, serializedKeyGroupStates, null);
        coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, statefulExec1.getAttemptId(), checkpointId, new CheckpointMetrics(), checkpointStateHandles));
        coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, statefulExec2.getAttemptId(), checkpointId, new CheckpointMetrics(), checkpointStateHandles));
        coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, statefulExec3.getAttemptId(), checkpointId, new CheckpointMetrics(), checkpointStateHandles));
        coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, statelessExec1.getAttemptId(), checkpointId));
        coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, statelessExec2.getAttemptId(), checkpointId));
        assertEquals(1, coord.getNumberOfRetainedSuccessfulCheckpoints());
        assertEquals(0, coord.getNumberOfPendingCheckpoints());
        // let the coordinator inject the state
        coord.restoreLatestCheckpointedState(map, true, false);
        // verify that each stateful vertex got the state
        final TaskStateHandles taskStateHandles = new TaskStateHandles(serializedState, Collections.<Collection<OperatorStateHandle>>singletonList(null), Collections.<Collection<OperatorStateHandle>>singletonList(null), Collections.singletonList(serializedKeyGroupStates), null);
        BaseMatcher<TaskStateHandles> matcher = new BaseMatcher<TaskStateHandles>() {

            @Override
            public boolean matches(Object o) {
                if (o instanceof TaskStateHandles) {
                    return o.equals(taskStateHandles);
                }
                return false;
            }

            @Override
            public void describeTo(Description description) {
                description.appendValue(taskStateHandles);
            }
        };
        verify(statefulExec1, times(1)).setInitialState(Mockito.argThat(matcher));
        verify(statefulExec2, times(1)).setInitialState(Mockito.argThat(matcher));
        verify(statefulExec3, times(1)).setInitialState(Mockito.argThat(matcher));
        verify(statelessExec1, times(0)).setInitialState(Mockito.<TaskStateHandles>any());
        verify(statelessExec2, times(0)).setInitialState(Mockito.<TaskStateHandles>any());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}