Examples with TestingSchedulingTopology org.apache.flink.runtime.scheduler.strategy.TestingSchedulingTopology used on opensource projects

Example 1 with TestingSchedulingTopology

use of org.apache.flink.runtime.scheduler.strategy.TestingSchedulingTopology in project flink by apache.

the class ExecutionFailureHandlerTest method setUp.

@Before
public void setUp() {
    TestingSchedulingTopology topology = new TestingSchedulingTopology();
    topology.newExecutionVertex();
    schedulingTopology = topology;
    failoverStrategy = new TestFailoverStrategy();
    backoffTimeStrategy = new TestRestartBackoffTimeStrategy(true, RESTART_DELAY_MS);
    executionFailureHandler = new ExecutionFailureHandler(schedulingTopology, failoverStrategy, backoffTimeStrategy);
}

Example 2 with TestingSchedulingTopology

use of org.apache.flink.runtime.scheduler.strategy.TestingSchedulingTopology in project flink by apache.

the class RestartAllFailoverStrategyTest method testGetTasksNeedingRestart.

@Test
public void testGetTasksNeedingRestart() {
    final TestingSchedulingTopology topology = new TestingSchedulingTopology();
    final TestingSchedulingExecutionVertex v1 = topology.newExecutionVertex();
    final TestingSchedulingExecutionVertex v2 = topology.newExecutionVertex();
    final TestingSchedulingExecutionVertex v3 = topology.newExecutionVertex();
    topology.connect(v1, v2, ResultPartitionType.PIPELINED);
    topology.connect(v2, v3, ResultPartitionType.BLOCKING);
    final RestartAllFailoverStrategy strategy = new RestartAllFailoverStrategy(topology);
    assertEquals(new HashSet<>(Arrays.asList(v1.getId(), v2.getId(), v3.getId())), strategy.getTasksNeedingRestart(v1.getId(), new Exception("Test failure")));
}

Example 3 with TestingSchedulingTopology

use of org.apache.flink.runtime.scheduler.strategy.TestingSchedulingTopology in project flink by apache.

the class RestartPipelinedRegionFailoverStrategyTest method testRegionFailoverForVariousResultPartitionAvailabilityCombinations.

/**
 * Tests to verify region failover results regarding different input result partition
 * availability combinations.
 *
 * <pre>
 *     (v1) --rp1--\
 *                 (v3)
 *     (v2) --rp2--/
 *
 *             ^
 *             |
 *         (blocking)
 * </pre>
 *
 * Each vertex is in an individual region. rp1, rp2 are result partitions.
 */
@Test
public void testRegionFailoverForVariousResultPartitionAvailabilityCombinations() throws Exception {
    TestingSchedulingTopology topology = new TestingSchedulingTopology();
    TestingSchedulingExecutionVertex v1 = topology.newExecutionVertex(ExecutionState.FINISHED);
    TestingSchedulingExecutionVertex v2 = topology.newExecutionVertex(ExecutionState.FINISHED);
    TestingSchedulingExecutionVertex v3 = topology.newExecutionVertex(ExecutionState.RUNNING);
    topology.connect(v1, v3, ResultPartitionType.BLOCKING);
    topology.connect(v2, v3, ResultPartitionType.BLOCKING);
    TestResultPartitionAvailabilityChecker availabilityChecker = new TestResultPartitionAvailabilityChecker();
    RestartPipelinedRegionFailoverStrategy strategy = new RestartPipelinedRegionFailoverStrategy(topology, availabilityChecker);
    IntermediateResultPartitionID rp1ID = v1.getProducedResults().iterator().next().getId();
    IntermediateResultPartitionID rp2ID = v2.getProducedResults().iterator().next().getId();
    // -------------------------------------------------
    // Combination1: (rp1 == available, rp2 == available)
    // -------------------------------------------------
    availabilityChecker.failedPartitions.clear();
    verifyThatFailedExecution(strategy, v1).restarts(v1, v3);
    verifyThatFailedExecution(strategy, v2).restarts(v2, v3);
    verifyThatFailedExecution(strategy, v3).restarts(v3);
    // -------------------------------------------------
    // Combination2: (rp1 == unavailable, rp2 == available)
    // -------------------------------------------------
    availabilityChecker.failedPartitions.clear();
    availabilityChecker.markResultPartitionFailed(rp1ID);
    verifyThatFailedExecution(strategy, v1).restarts(v1, v3);
    verifyThatFailedExecution(strategy, v2).restarts(v1, v2, v3);
    verifyThatFailedExecution(strategy, v3).restarts(v1, v3);
    // -------------------------------------------------
    // Combination3: (rp1 == available, rp2 == unavailable)
    // -------------------------------------------------
    availabilityChecker.failedPartitions.clear();
    availabilityChecker.markResultPartitionFailed(rp2ID);
    verifyThatFailedExecution(strategy, v1).restarts(v1, v2, v3);
    verifyThatFailedExecution(strategy, v2).restarts(v2, v3);
    verifyThatFailedExecution(strategy, v3).restarts(v2, v3);
    // -------------------------------------------------
    // Combination4: (rp1 == unavailable, rp == unavailable)
    // -------------------------------------------------
    availabilityChecker.failedPartitions.clear();
    availabilityChecker.markResultPartitionFailed(rp1ID);
    availabilityChecker.markResultPartitionFailed(rp2ID);
    verifyThatFailedExecution(strategy, v1).restarts(v1, v2, v3);
    verifyThatFailedExecution(strategy, v2).restarts(v1, v2, v3);
    verifyThatFailedExecution(strategy, v3).restarts(v1, v2, v3);
}

Example 4 with TestingSchedulingTopology

use of org.apache.flink.runtime.scheduler.strategy.TestingSchedulingTopology in project flink by apache.

the class RestartPipelinedRegionFailoverStrategyTest method testRegionFailoverDoesNotRestartCreatedExecutions.

/**
 * Tests region failover does not restart vertexes which are already in initial CREATED state.
 *
 * <pre>
 *     (v1) --|--> (v2)
 *
 *            ^
 *            |
 *       (blocking)
 * </pre>
 *
 * Component 1: 1; component 2: 2
 */
@Test
public void testRegionFailoverDoesNotRestartCreatedExecutions() {
    TestingSchedulingTopology topology = new TestingSchedulingTopology();
    TestingSchedulingExecutionVertex v1 = topology.newExecutionVertex(ExecutionState.CREATED);
    TestingSchedulingExecutionVertex v2 = topology.newExecutionVertex(ExecutionState.CREATED);
    topology.connect(v1, v2, ResultPartitionType.BLOCKING);
    FailoverStrategy strategy = new RestartPipelinedRegionFailoverStrategy(topology);
    verifyThatFailedExecution(strategy, v2).restarts();
    TestingSchedulingResultPartition v1out = v2.getConsumedResults().iterator().next();
    verifyThatFailedExecution(strategy, v2).partitionConnectionCause(v1out).restarts();
}

Example 5 with TestingSchedulingTopology

use of org.apache.flink.runtime.scheduler.strategy.TestingSchedulingTopology in project flink by apache.

the class SchedulingPipelinedRegionComputeUtilTest method testPipelinedApproximateDifferentRegions.

@Test
public void testPipelinedApproximateDifferentRegions() {
    TestingSchedulingTopology topology = new TestingSchedulingTopology();
    TestingSchedulingExecutionVertex v1 = topology.newExecutionVertex();
    TestingSchedulingExecutionVertex v2 = topology.newExecutionVertex();
    TestingSchedulingExecutionVertex v3 = topology.newExecutionVertex();
    TestingSchedulingExecutionVertex v4 = topology.newExecutionVertex();
    topology.connect(v1, v2, ResultPartitionType.PIPELINED_APPROXIMATE).connect(v1, v3, ResultPartitionType.PIPELINED_APPROXIMATE).connect(v2, v4, ResultPartitionType.PIPELINED_APPROXIMATE).connect(v3, v4, ResultPartitionType.PIPELINED_APPROXIMATE);
    Map<ExecutionVertexID, Set<SchedulingExecutionVertex>> pipelinedRegionByVertex = computePipelinedRegionByVertex(topology);
    Set<SchedulingExecutionVertex> r1 = pipelinedRegionByVertex.get(v1.getId());
    Set<SchedulingExecutionVertex> r2 = pipelinedRegionByVertex.get(v2.getId());
    Set<SchedulingExecutionVertex> r3 = pipelinedRegionByVertex.get(v3.getId());
    Set<SchedulingExecutionVertex> r4 = pipelinedRegionByVertex.get(v4.getId());
    assertDistinctRegions(r1, r2, r3, r4);
}