Examples with DefaultLogicalPipelinedRegion org.apache.flink.runtime.jobgraph.topology.DefaultLogicalPipelinedRegion used on opensource projects

Example 1 with DefaultLogicalPipelinedRegion

use of org.apache.flink.runtime.jobgraph.topology.DefaultLogicalPipelinedRegion in project flink by apache.

the class DefaultExecutionTopology method computeLogicalPipelinedRegionsByJobVertexId.

private static Map<JobVertexID, DefaultLogicalPipelinedRegion> computeLogicalPipelinedRegionsByJobVertexId(final ExecutionGraph executionGraph) {
    List<JobVertex> topologicallySortedJobVertices = IterableUtils.toStream(executionGraph.getVerticesTopologically()).map(ExecutionJobVertex::getJobVertex).collect(Collectors.toList());
    Iterable<DefaultLogicalPipelinedRegion> logicalPipelinedRegions = DefaultLogicalTopology.fromTopologicallySortedJobVertices(topologicallySortedJobVertices).getAllPipelinedRegions();
    Map<JobVertexID, DefaultLogicalPipelinedRegion> logicalPipelinedRegionsByJobVertexId = new HashMap<>();
    for (DefaultLogicalPipelinedRegion logicalPipelinedRegion : logicalPipelinedRegions) {
        for (LogicalVertex vertex : logicalPipelinedRegion.getVertices()) {
            logicalPipelinedRegionsByJobVertexId.put(vertex.getId(), logicalPipelinedRegion);
        }
    }
    return logicalPipelinedRegionsByJobVertexId;
}

Example 2 with DefaultLogicalPipelinedRegion

use of org.apache.flink.runtime.jobgraph.topology.DefaultLogicalPipelinedRegion in project flink by apache.

the class DefaultExecutionTopology method generateNewPipelinedRegions.

private void generateNewPipelinedRegions(Iterable<ExecutionVertex> newExecutionVertices) {
    final Iterable<DefaultExecutionVertex> newSchedulingExecutionVertices = IterableUtils.toStream(newExecutionVertices).map(ExecutionVertex::getID).map(executionVerticesById::get).collect(Collectors.toList());
    Map<DefaultLogicalPipelinedRegion, List<DefaultExecutionVertex>> sortedExecutionVerticesInPipelinedRegion = new IdentityHashMap<>();
    for (DefaultExecutionVertex schedulingVertex : newSchedulingExecutionVertices) {
        sortedExecutionVerticesInPipelinedRegion.computeIfAbsent(logicalPipelinedRegionsByJobVertexId.get(schedulingVertex.getId().getJobVertexId()), ignore -> new ArrayList<>()).add(schedulingVertex);
    }
    long buildRegionsStartTime = System.nanoTime();
    Set<Set<SchedulingExecutionVertex>> rawPipelinedRegions = Collections.newSetFromMap(new IdentityHashMap<>());
    // SchedulingPipelinedRegions are both connected with inter-region blocking edges.
    for (Map.Entry<DefaultLogicalPipelinedRegion, List<DefaultExecutionVertex>> entry : sortedExecutionVerticesInPipelinedRegion.entrySet()) {
        DefaultLogicalPipelinedRegion logicalPipelinedRegion = entry.getKey();
        List<DefaultExecutionVertex> schedulingExecutionVertices = entry.getValue();
        if (containsIntraRegionAllToAllEdge(logicalPipelinedRegion)) {
            // For edges inside one LogicalPipelinedRegion, if there is any all-to-all edge, it
            // could be under two circumstances:
            // 
            // 1. Pipelined all-to-all edge:
            // Pipelined all-to-all edge will connect all vertices pipelined. Therefore,
            // all execution vertices derived from this LogicalPipelinedRegion should be in one
            // SchedulingPipelinedRegion.
            // 
            // 2. Blocking all-to-all edge:
            // For intra-region blocking all-to-all edge, we must make sure all the vertices
            // are inside one SchedulingPipelinedRegion, so that there will be no deadlock
            // happens during scheduling. For more details about this case, please refer to
            // FLINK-17330 (https://issues.apache.org/jira/browse/FLINK-17330).
            // 
            // Therefore, if a LogicalPipelinedRegion contains any intra-region all-to-all
            // edge, we just convert the entire LogicalPipelinedRegion to a sole
            // SchedulingPipelinedRegion directly.
            rawPipelinedRegions.add(new HashSet<>(schedulingExecutionVertices));
        } else {
            // If there are only pointwise edges inside the LogicalPipelinedRegion, we can use
            // SchedulingPipelinedRegionComputeUtil to compute the regions with O(N) computation
            // complexity.
            rawPipelinedRegions.addAll(SchedulingPipelinedRegionComputeUtil.computePipelinedRegions(schedulingExecutionVertices, executionVerticesById::get, resultPartitionsById::get));
        }
    }
    for (Set<? extends SchedulingExecutionVertex> rawPipelinedRegion : rawPipelinedRegions) {
        // noinspection unchecked
        final DefaultSchedulingPipelinedRegion pipelinedRegion = new DefaultSchedulingPipelinedRegion((Set<DefaultExecutionVertex>) rawPipelinedRegion, resultPartitionsById::get);
        pipelinedRegions.add(pipelinedRegion);
        for (SchedulingExecutionVertex executionVertex : rawPipelinedRegion) {
            pipelinedRegionsByVertex.put(executionVertex.getId(), pipelinedRegion);
        }
    }
    long buildRegionsDuration = (System.nanoTime() - buildRegionsStartTime) / 1_000_000;
    LOG.info("Built {} new pipelined regions in {} ms, total {} pipelined regions currently.", rawPipelinedRegions.size(), buildRegionsDuration, pipelinedRegions.size());
}

Example 3 with DefaultLogicalPipelinedRegion

use of org.apache.flink.runtime.jobgraph.topology.DefaultLogicalPipelinedRegion in project flink by apache.

the class StreamingJobGraphGenerator method buildVertexRegionSlotSharingGroups.

/**
 * Maps a vertex to its region slot sharing group. If {@link
 * StreamGraph#isAllVerticesInSameSlotSharingGroupByDefault()} returns true, all regions will be
 * in the same slot sharing group.
 */
private Map<JobVertexID, SlotSharingGroup> buildVertexRegionSlotSharingGroups() {
    final Map<JobVertexID, SlotSharingGroup> vertexRegionSlotSharingGroups = new HashMap<>();
    final SlotSharingGroup defaultSlotSharingGroup = new SlotSharingGroup();
    streamGraph.getSlotSharingGroupResource(StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP).ifPresent(defaultSlotSharingGroup::setResourceProfile);
    final boolean allRegionsInSameSlotSharingGroup = streamGraph.isAllVerticesInSameSlotSharingGroupByDefault();
    final Iterable<DefaultLogicalPipelinedRegion> regions = DefaultLogicalTopology.fromJobGraph(jobGraph).getAllPipelinedRegions();
    for (DefaultLogicalPipelinedRegion region : regions) {
        final SlotSharingGroup regionSlotSharingGroup;
        if (allRegionsInSameSlotSharingGroup) {
            regionSlotSharingGroup = defaultSlotSharingGroup;
        } else {
            regionSlotSharingGroup = new SlotSharingGroup();
            streamGraph.getSlotSharingGroupResource(StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP).ifPresent(regionSlotSharingGroup::setResourceProfile);
        }
        for (LogicalVertex vertex : region.getVertices()) {
            vertexRegionSlotSharingGroups.put(vertex.getId(), regionSlotSharingGroup);
        }
    }
    return vertexRegionSlotSharingGroups;
}