Examples with ExecutionVertexID org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID used on opensource projects

Example 56 with ExecutionVertexID

use of org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID in project flink-mirror by flink-ci.

the class SchedulingPipelinedRegionComputeUtilTest method testEmbarrassinglyParallelCase.

/**
 * Tests that validates that embarrassingly parallel chains of vertices work correctly.
 *
 * <pre>
 *     (a1) --> (b1)
 *
 *     (a2) --> (b2)
 *
 *     (a3) --> (b3)
 * </pre>
 */
@Test
public void testEmbarrassinglyParallelCase() {
    TestingSchedulingTopology topology = new TestingSchedulingTopology();
    TestingSchedulingExecutionVertex va1 = topology.newExecutionVertex();
    TestingSchedulingExecutionVertex va2 = topology.newExecutionVertex();
    TestingSchedulingExecutionVertex va3 = topology.newExecutionVertex();
    TestingSchedulingExecutionVertex vb1 = topology.newExecutionVertex();
    TestingSchedulingExecutionVertex vb2 = topology.newExecutionVertex();
    TestingSchedulingExecutionVertex vb3 = topology.newExecutionVertex();
    topology.connect(va1, vb1, ResultPartitionType.PIPELINED).connect(va2, vb2, ResultPartitionType.PIPELINED).connect(va3, vb3, ResultPartitionType.PIPELINED);
    Map<ExecutionVertexID, Set<SchedulingExecutionVertex>> pipelinedRegionByVertex = computePipelinedRegionByVertex(topology);
    Set<SchedulingExecutionVertex> ra1 = pipelinedRegionByVertex.get(va1.getId());
    Set<SchedulingExecutionVertex> ra2 = pipelinedRegionByVertex.get(va2.getId());
    Set<SchedulingExecutionVertex> ra3 = pipelinedRegionByVertex.get(va3.getId());
    Set<SchedulingExecutionVertex> rb1 = pipelinedRegionByVertex.get(vb1.getId());
    Set<SchedulingExecutionVertex> rb2 = pipelinedRegionByVertex.get(vb2.getId());
    Set<SchedulingExecutionVertex> rb3 = pipelinedRegionByVertex.get(vb3.getId());
    assertSameRegion(ra1, rb1);
    assertSameRegion(ra2, rb2);
    assertSameRegion(ra3, rb3);
    assertDistinctRegions(ra1, ra2, ra3);
}

Example 57 with ExecutionVertexID

use of org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID in project flink-mirror by flink-ci.

the class SchedulingPipelinedRegionComputeUtilTest method testMultipleComponentsViaCascadeOfJoins.

/**
 * Cascades of joins with partially blocking, partially pipelined exchanges.
 *
 * <pre>
 *     (1)--+
 *          +--(5)-+
 *     (2)--+      |
 *              (blocking)
 *                 |
 *                 +--(7)
 *                 |
 *              (blocking)
 *     (3)--+      |
 *          +--(6)-+
 *     (4)--+
 * </pre>
 *
 * <p>Component 1: 1, 2, 5; component 2: 3,4,6; component 3: 7
 */
@Test
public void testMultipleComponentsViaCascadeOfJoins() {
    TestingSchedulingTopology topology = new TestingSchedulingTopology();
    TestingSchedulingExecutionVertex v1 = topology.newExecutionVertex();
    TestingSchedulingExecutionVertex v2 = topology.newExecutionVertex();
    TestingSchedulingExecutionVertex v3 = topology.newExecutionVertex();
    TestingSchedulingExecutionVertex v4 = topology.newExecutionVertex();
    TestingSchedulingExecutionVertex v5 = topology.newExecutionVertex();
    TestingSchedulingExecutionVertex v6 = topology.newExecutionVertex();
    TestingSchedulingExecutionVertex v7 = topology.newExecutionVertex();
    topology.connect(v1, v5, ResultPartitionType.PIPELINED).connect(v2, v5, ResultPartitionType.PIPELINED).connect(v3, v6, ResultPartitionType.PIPELINED).connect(v4, v6, ResultPartitionType.PIPELINED).connect(v5, v7, ResultPartitionType.BLOCKING).connect(v6, v7, ResultPartitionType.BLOCKING);
    Map<ExecutionVertexID, Set<SchedulingExecutionVertex>> pipelinedRegionByVertex = computePipelinedRegionByVertex(topology);
    Set<SchedulingExecutionVertex> r1 = pipelinedRegionByVertex.get(v1.getId());
    Set<SchedulingExecutionVertex> r2 = pipelinedRegionByVertex.get(v2.getId());
    Set<SchedulingExecutionVertex> r3 = pipelinedRegionByVertex.get(v3.getId());
    Set<SchedulingExecutionVertex> r4 = pipelinedRegionByVertex.get(v4.getId());
    Set<SchedulingExecutionVertex> r5 = pipelinedRegionByVertex.get(v5.getId());
    Set<SchedulingExecutionVertex> r6 = pipelinedRegionByVertex.get(v6.getId());
    Set<SchedulingExecutionVertex> r7 = pipelinedRegionByVertex.get(v7.getId());
    assertSameRegion(r1, r2, r5);
    assertSameRegion(r3, r4, r6);
    assertDistinctRegions(r1, r3, r7);
}

Example 58 with ExecutionVertexID

use of org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID in project flink-mirror by flink-ci.

the class SchedulingPipelinedRegionComputeUtilTest method testTwoComponentsViaBlockingExchange.

/**
 * Tests the below topology.
 *
 * <pre>
 *     (a1) -+-> (b1) -+-> (c1)
 *           X
 *     (a2) -+-> (b2) -+-> (c2)
 *
 *           ^         ^
 *           |         |
 *     (pipelined) (blocking)
 * </pre>
 */
@Test
public void testTwoComponentsViaBlockingExchange() {
    TestingSchedulingTopology topology = new TestingSchedulingTopology();
    TestingSchedulingExecutionVertex va1 = topology.newExecutionVertex();
    TestingSchedulingExecutionVertex va2 = topology.newExecutionVertex();
    TestingSchedulingExecutionVertex vb1 = topology.newExecutionVertex();
    TestingSchedulingExecutionVertex vb2 = topology.newExecutionVertex();
    TestingSchedulingExecutionVertex vc1 = topology.newExecutionVertex();
    TestingSchedulingExecutionVertex vc2 = topology.newExecutionVertex();
    topology.connect(va1, vb1, ResultPartitionType.PIPELINED).connect(va1, vb2, ResultPartitionType.PIPELINED).connect(va2, vb1, ResultPartitionType.PIPELINED).connect(va2, vb2, ResultPartitionType.PIPELINED).connect(vb1, vc1, ResultPartitionType.BLOCKING).connect(vb2, vc2, ResultPartitionType.BLOCKING);
    Map<ExecutionVertexID, Set<SchedulingExecutionVertex>> pipelinedRegionByVertex = computePipelinedRegionByVertex(topology);
    Set<SchedulingExecutionVertex> ra1 = pipelinedRegionByVertex.get(va1.getId());
    Set<SchedulingExecutionVertex> ra2 = pipelinedRegionByVertex.get(va2.getId());
    Set<SchedulingExecutionVertex> rb1 = pipelinedRegionByVertex.get(vb1.getId());
    Set<SchedulingExecutionVertex> rb2 = pipelinedRegionByVertex.get(vb2.getId());
    Set<SchedulingExecutionVertex> rc1 = pipelinedRegionByVertex.get(vc1.getId());
    Set<SchedulingExecutionVertex> rc2 = pipelinedRegionByVertex.get(vc2.getId());
    assertSameRegion(ra1, ra2, rb1, rb2);
    assertDistinctRegions(ra1, rc1, rc2);
}

Example 59 with ExecutionVertexID

use of org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID in project flink-mirror by flink-ci.

the class DefaultExecutionGraph method maybeReleasePartitionGroupsFor.

private void maybeReleasePartitionGroupsFor(final Execution attempt) {
    final ExecutionVertexID finishedExecutionVertex = attempt.getVertex().getID();
    if (attempt.getState() == ExecutionState.FINISHED) {
        final List<ConsumedPartitionGroup> releasablePartitionGroups = partitionGroupReleaseStrategy.vertexFinished(finishedExecutionVertex);
        releasePartitionGroups(releasablePartitionGroups);
    } else {
        partitionGroupReleaseStrategy.vertexUnfinished(finishedExecutionVertex);
    }
}

Example 60 with ExecutionVertexID

use of org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID in project flink-mirror by flink-ci.

the class DefaultExecutionTopology method generateNewExecutionVerticesAndResultPartitions.

private void generateNewExecutionVerticesAndResultPartitions(Iterable<ExecutionVertex> newExecutionVertices) {
    for (ExecutionVertex vertex : newExecutionVertices) {
        List<DefaultResultPartition> producedPartitions = generateProducedSchedulingResultPartition(vertex.getProducedPartitions(), edgeManager::getConsumerVertexGroupForPartition);
        producedPartitions.forEach(partition -> resultPartitionsById.put(partition.getId(), partition));
        DefaultExecutionVertex schedulingVertex = generateSchedulingExecutionVertex(vertex, producedPartitions, edgeManager.getConsumedPartitionGroupsForVertex(vertex.getID()), resultPartitionsById::get);
        executionVerticesById.put(schedulingVertex.getId(), schedulingVertex);
    }
    executionVerticesList.clear();
    for (ExecutionVertexID vertexID : sortedExecutionVertexIds.get()) {
        executionVerticesList.add(executionVerticesById.get(vertexID));
    }
}