Examples with TaskManagerLocation org.apache.flink.runtime.taskmanager.TaskManagerLocation used on opensource projects

Example 31 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class JobVertexDetailsHandlerTest method compareVertexDetails.

private static void compareVertexDetails(AccessExecutionJobVertex originalTask, String json) throws IOException {
    JsonNode result = ArchivedJobGenerationUtils.mapper.readTree(json);
    Assert.assertEquals(originalTask.getJobVertexId().toString(), result.get("id").asText());
    Assert.assertEquals(originalTask.getName(), result.get("name").asText());
    Assert.assertEquals(originalTask.getParallelism(), result.get("parallelism").asInt());
    Assert.assertTrue(result.get("now").asLong() > 0);
    ArrayNode subtasks = (ArrayNode) result.get("subtasks");
    Assert.assertEquals(originalTask.getTaskVertices().length, subtasks.size());
    for (int x = 0; x < originalTask.getTaskVertices().length; x++) {
        AccessExecutionVertex expectedSubtask = originalTask.getTaskVertices()[x];
        JsonNode subtask = subtasks.get(x);
        Assert.assertEquals(x, subtask.get("subtask").asInt());
        Assert.assertEquals(expectedSubtask.getExecutionState().name(), subtask.get("status").asText());
        Assert.assertEquals(expectedSubtask.getCurrentExecutionAttempt().getAttemptNumber(), subtask.get("attempt").asInt());
        TaskManagerLocation location = expectedSubtask.getCurrentAssignedResourceLocation();
        String expectedLocationString = location.getHostname() + ":" + location.dataPort();
        Assert.assertEquals(expectedLocationString, subtask.get("host").asText());
        long start = expectedSubtask.getStateTimestamp(ExecutionState.DEPLOYING);
        Assert.assertEquals(start, subtask.get("start-time").asLong());
        long end = expectedSubtask.getStateTimestamp(ExecutionState.FINISHED);
        Assert.assertEquals(end, subtask.get("end-time").asLong());
        Assert.assertEquals(end - start, subtask.get("duration").asLong());
        ArchivedJobGenerationUtils.compareIoMetrics(expectedSubtask.getCurrentExecutionAttempt().getIOMetrics(), subtask.get("metrics"));
    }
}

Example 32 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class ExecutionVertex method getPreferredLocationsBasedOnInputs.

/**
	 * Gets the location preferences of the vertex's current task execution, as determined by the locations
	 * of the predecessors from which it receives input data.
	 * If there are more than MAX_DISTINCT_LOCATIONS_TO_CONSIDER different locations of source data, this
	 * method returns {@code null} to indicate no location preference.
	 *
	 * @return The preferred locations based in input streams, or an empty iterable,
	 *         if there is no input-based preference.
	 */
public Iterable<TaskManagerLocation> getPreferredLocationsBasedOnInputs() {
    // otherwise, base the preferred locations on the input connections
    if (inputEdges == null) {
        return Collections.emptySet();
    } else {
        Set<TaskManagerLocation> locations = new HashSet<>();
        Set<TaskManagerLocation> inputLocations = new HashSet<>();
        // go over all inputs
        for (int i = 0; i < inputEdges.length; i++) {
            inputLocations.clear();
            ExecutionEdge[] sources = inputEdges[i];
            if (sources != null) {
                // go over all input sources
                for (int k = 0; k < sources.length; k++) {
                    // look-up assigned slot of input source
                    SimpleSlot sourceSlot = sources[k].getSource().getProducer().getCurrentAssignedResource();
                    if (sourceSlot != null) {
                        // add input location
                        inputLocations.add(sourceSlot.getTaskManagerLocation());
                        // inputs which have too many distinct sources are not considered
                        if (inputLocations.size() > MAX_DISTINCT_LOCATIONS_TO_CONSIDER) {
                            inputLocations.clear();
                            break;
                        }
                    }
                }
            }
            // keep the locations of the input with the least preferred locations
            if (// nothing assigned yet
            locations.isEmpty() || (!inputLocations.isEmpty() && inputLocations.size() < locations.size())) {
                // current input has fewer preferred locations
                locations.clear();
                locations.addAll(inputLocations);
            }
        }
        return locations.isEmpty() ? Collections.<TaskManagerLocation>emptyList() : locations;
    }
}

Example 33 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class InputChannelDeploymentDescriptor method fromEdges.

// ------------------------------------------------------------------------
/**
	 * Creates an input channel deployment descriptor for each partition.
	 */
public static InputChannelDeploymentDescriptor[] fromEdges(ExecutionEdge[] edges, SimpleSlot consumerSlot, boolean allowLazyDeployment) throws ExecutionGraphException {
    final ResourceID consumerTaskManager = consumerSlot.getTaskManagerID();
    final InputChannelDeploymentDescriptor[] icdd = new InputChannelDeploymentDescriptor[edges.length];
    // Each edge is connected to a different result partition
    for (int i = 0; i < edges.length; i++) {
        final IntermediateResultPartition consumedPartition = edges[i].getSource();
        final Execution producer = consumedPartition.getProducer().getCurrentExecutionAttempt();
        final ExecutionState producerState = producer.getState();
        final SimpleSlot producerSlot = producer.getAssignedResource();
        final ResultPartitionLocation partitionLocation;
        // The producing task needs to be RUNNING or already FINISHED
        if (consumedPartition.isConsumable() && producerSlot != null && (producerState == ExecutionState.RUNNING || producerState == ExecutionState.FINISHED || producerState == ExecutionState.SCHEDULED || producerState == ExecutionState.DEPLOYING)) {
            final TaskManagerLocation partitionTaskManagerLocation = producerSlot.getTaskManagerLocation();
            final ResourceID partitionTaskManager = partitionTaskManagerLocation.getResourceID();
            if (partitionTaskManager.equals(consumerTaskManager)) {
                // Consuming task is deployed to the same TaskManager as the partition => local
                partitionLocation = ResultPartitionLocation.createLocal();
            } else {
                // Different instances => remote
                final ConnectionID connectionId = new ConnectionID(partitionTaskManagerLocation, consumedPartition.getIntermediateResult().getConnectionIndex());
                partitionLocation = ResultPartitionLocation.createRemote(connectionId);
            }
        } else if (allowLazyDeployment) {
            // The producing task might not have registered the partition yet
            partitionLocation = ResultPartitionLocation.createUnknown();
        } else if (producerState == ExecutionState.CANCELING || producerState == ExecutionState.CANCELED || producerState == ExecutionState.FAILED) {
            String msg = "Trying to schedule a task whose inputs were canceled or failed. " + "The producer is in state " + producerState + ".";
            throw new ExecutionGraphException(msg);
        } else {
            String msg = String.format("Trying to eagerly schedule a task whose inputs " + "are not ready (partition consumable? %s, producer state: %s, producer slot: %s).", consumedPartition.isConsumable(), producerState, producerSlot);
            throw new ExecutionGraphException(msg);
        }
        final ResultPartitionID consumedPartitionId = new ResultPartitionID(consumedPartition.getPartitionId(), producer.getAttemptId());
        icdd[i] = new InputChannelDeploymentDescriptor(consumedPartitionId, partitionLocation);
    }
    return icdd;
}

Example 34 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class PartialInputChannelDeploymentDescriptor method createInputChannelDeploymentDescriptor.

/**
	 * Creates a channel deployment descriptor by completing the partition location.
	 *
	 * @see InputChannelDeploymentDescriptor
	 */
public InputChannelDeploymentDescriptor createInputChannelDeploymentDescriptor(Execution consumerExecution) {
    checkNotNull(consumerExecution, "consumerExecution");
    TaskManagerLocation consumerLocation = consumerExecution.getAssignedResourceLocation();
    checkNotNull(consumerLocation, "Consumer connection info null");
    final ResultPartitionLocation partitionLocation;
    if (consumerLocation.equals(partitionTaskManagerLocation)) {
        partitionLocation = ResultPartitionLocation.createLocal();
    } else {
        partitionLocation = ResultPartitionLocation.createRemote(new ConnectionID(partitionTaskManagerLocation, partitionConnectionIndex));
    }
    return new InputChannelDeploymentDescriptor(partitionID, partitionLocation);
}

Example 35 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class PartialInputChannelDeploymentDescriptor method fromEdge.

// ------------------------------------------------------------------------
/**
	 * Creates a partial input channel for the given partition and producing task.
	 */
public static PartialInputChannelDeploymentDescriptor fromEdge(IntermediateResultPartition partition, Execution producer) {
    final ResultPartitionID partitionId = new ResultPartitionID(partition.getPartitionId(), producer.getAttemptId());
    final IntermediateResult result = partition.getIntermediateResult();
    final IntermediateDataSetID resultId = result.getId();
    final TaskManagerLocation partitionConnectionInfo = producer.getAssignedResourceLocation();
    final int partitionConnectionIndex = result.getConnectionIndex();
    return new PartialInputChannelDeploymentDescriptor(resultId, partitionId, partitionConnectionInfo, partitionConnectionIndex);
}