Examples with BucketNodeMap com.facebook.presto.execution.scheduler.BucketNodeMap used on opensource projects

Example 1 with BucketNodeMap

use of com.facebook.presto.execution.scheduler.BucketNodeMap in project presto by prestodb.

the class TestDynamicLifespanScheduler method testAffinityScheduleFailedLocality.

@Test
public void testAffinityScheduleFailedLocality() {
    BucketNodeMap bucketNodeMap = new DynamicBucketNodeMap(split -> ((TestDynamicLifespanScheduler.TestSplit) split.getConnectorSplit()).getBucketNumber(), BUCKET_COUNT, ImmutableList.of(node1, node2, node1, node2, node1, node2, node1, node2, node1, node2));
    LifespanScheduler lifespanScheduler = getAffinityLifespanScheduler(bucketNodeMap);
    TestDynamicLifespanScheduler.TestingSourceScheduler sourceScheduler = new TestDynamicLifespanScheduler.TestingSourceScheduler();
    lifespanScheduler.scheduleInitial(sourceScheduler);
    lifespanScheduler.onLifespanExecutionFinished(ImmutableList.of(sourceScheduler.getLastStartedLifespans().get(1)));
    assertEquals(sourceScheduler.getLastStartedLifespans().size(), 2);
    sourceScheduler.getLastStartedLifespans().clear();
    lifespanScheduler.onTaskFailed(0, ImmutableList.of(sourceScheduler));
    assertEquals(sourceScheduler.getLastRewoundLifespans().size(), 1);
    sourceScheduler.getLastRewoundLifespans().clear();
    while (!lifespanScheduler.allLifespanExecutionFinished()) {
        lifespanScheduler.schedule(sourceScheduler);
        lifespanScheduler.onLifespanExecutionFinished(sourceScheduler.getLastStartedLifespans());
        assertEquals(sourceScheduler.getLastStartedLifespans().size(), 1);
        sourceScheduler.getLastStartedLifespans().clear();
    }
    assertEquals(bucketNodeMap.getAssignedNode(0).get(), node2);
    // bucket 1 is already scheduled, thus its assignedNode is changed
    assertEquals(bucketNodeMap.getAssignedNode(1).get(), node2);
    assertEquals(bucketNodeMap.getAssignedNode(2).get(), node2);
    // bucket 3 is not scheduled yet, thus its assignedNode remains
    assertEquals(bucketNodeMap.getAssignedNode(3).get(), node2);
    assertEquals(bucketNodeMap.getAssignedNode(4).get(), node2);
    assertEquals(bucketNodeMap.getAssignedNode(5).get(), node2);
    assertEquals(bucketNodeMap.getAssignedNode(6).get(), node2);
    assertEquals(bucketNodeMap.getAssignedNode(7).get(), node2);
    assertEquals(bucketNodeMap.getAssignedNode(8).get(), node2);
    assertEquals(bucketNodeMap.getAssignedNode(9).get(), node2);
}

Example 2 with BucketNodeMap

use of com.facebook.presto.execution.scheduler.BucketNodeMap in project presto by prestodb.

the class TestDynamicLifespanScheduler method testAffinityScheduleLocality.

@Test
public void testAffinityScheduleLocality() {
    BucketNodeMap bucketNodeMap = new DynamicBucketNodeMap(split -> ((TestDynamicLifespanScheduler.TestSplit) split.getConnectorSplit()).getBucketNumber(), BUCKET_COUNT, ImmutableList.of(node1, node3, node1, node3, node1, node3, node1, node3, node1, node3));
    LifespanScheduler lifespanScheduler = getAffinityLifespanScheduler(bucketNodeMap);
    TestDynamicLifespanScheduler.TestingSourceScheduler sourceScheduler = new TestDynamicLifespanScheduler.TestingSourceScheduler();
    lifespanScheduler.scheduleInitial(sourceScheduler);
    assertEquals(bucketNodeMap.getAssignedNode(0).get(), node1);
    // bucket 1 is already scheduled, thus its assignedNode is changed
    assertEquals(bucketNodeMap.getAssignedNode(1).get(), node2);
    assertEquals(bucketNodeMap.getAssignedNode(2).get(), node1);
    // bucket 3 is not scheduled yet, thus its assignedNode remains
    assertEquals(bucketNodeMap.getAssignedNode(3).get(), node3);
    assertEquals(bucketNodeMap.getAssignedNode(4).get(), node1);
    assertEquals(bucketNodeMap.getAssignedNode(5).get(), node3);
    assertEquals(bucketNodeMap.getAssignedNode(6).get(), node1);
    assertEquals(bucketNodeMap.getAssignedNode(7).get(), node3);
    assertEquals(bucketNodeMap.getAssignedNode(8).get(), node1);
    assertEquals(bucketNodeMap.getAssignedNode(9).get(), node3);
    lifespanScheduler.onLifespanExecutionFinished(sourceScheduler.getLastStartedLifespans());
    assertEquals(sourceScheduler.getLastStartedLifespans().size(), 2);
    sourceScheduler.getLastStartedLifespans().clear();
    while (!lifespanScheduler.allLifespanExecutionFinished()) {
        lifespanScheduler.schedule(sourceScheduler);
        lifespanScheduler.onLifespanExecutionFinished(sourceScheduler.getLastStartedLifespans());
        assertEquals(sourceScheduler.getLastStartedLifespans().size(), 2);
        sourceScheduler.getLastStartedLifespans().clear();
    }
    assertEquals(bucketNodeMap.getAssignedNode(0).get(), node1);
    // bucket 1 is already scheduled, thus its assignedNode is changed
    assertEquals(bucketNodeMap.getAssignedNode(1).get(), node2);
    assertEquals(bucketNodeMap.getAssignedNode(2).get(), node1);
    // bucket 3 is not scheduled yet, thus its assignedNode remains
    assertEquals(bucketNodeMap.getAssignedNode(3).get(), node2);
    assertEquals(bucketNodeMap.getAssignedNode(4).get(), node1);
    assertEquals(bucketNodeMap.getAssignedNode(5).get(), node2);
    assertEquals(bucketNodeMap.getAssignedNode(6).get(), node1);
    assertEquals(bucketNodeMap.getAssignedNode(7).get(), node2);
    assertEquals(bucketNodeMap.getAssignedNode(8).get(), node1);
    assertEquals(bucketNodeMap.getAssignedNode(9).get(), node2);
}

Example 3 with BucketNodeMap

use of com.facebook.presto.execution.scheduler.BucketNodeMap in project presto by prestodb.

the class TestDynamicLifespanScheduler method testAffinityRetry.

@Test
public void testAffinityRetry() {
    BucketNodeMap bucketNodeMap = new DynamicBucketNodeMap(split -> ((TestDynamicLifespanScheduler.TestSplit) split.getConnectorSplit()).getBucketNumber(), BUCKET_COUNT, ImmutableList.of(node1, node2, node1, node2, node1, node2, node1, node2, node1, node2));
    LifespanScheduler lifespanScheduler = getAffinityLifespanScheduler(bucketNodeMap);
    TestDynamicLifespanScheduler.TestingSourceScheduler sourceScheduler = new TestDynamicLifespanScheduler.TestingSourceScheduler();
    lifespanScheduler.scheduleInitial(sourceScheduler);
    lifespanScheduler.onLifespanExecutionFinished(ImmutableList.of(sourceScheduler.getLastStartedLifespans().get(1)));
    assertEquals(sourceScheduler.getLastStartedLifespans().size(), 2);
    sourceScheduler.getLastStartedLifespans().clear();
    lifespanScheduler.onTaskFailed(0, ImmutableList.of(sourceScheduler));
    assertEquals(sourceScheduler.getLastRewoundLifespans().size(), 1);
    sourceScheduler.getLastRewoundLifespans().clear();
    while (!lifespanScheduler.allLifespanExecutionFinished()) {
        lifespanScheduler.schedule(sourceScheduler);
        lifespanScheduler.onLifespanExecutionFinished(sourceScheduler.getLastStartedLifespans());
        assertEquals(sourceScheduler.getLastStartedLifespans().size(), 1);
        sourceScheduler.getLastStartedLifespans().clear();
    }
}

Example 4 with BucketNodeMap

use of com.facebook.presto.execution.scheduler.BucketNodeMap in project presto by prestodb.

the class TestDynamicLifespanScheduler method testAffinitySchedule.

@Test
public void testAffinitySchedule() {
    BucketNodeMap bucketNodeMap = new DynamicBucketNodeMap(split -> ((TestDynamicLifespanScheduler.TestSplit) split.getConnectorSplit()).getBucketNumber(), BUCKET_COUNT, ImmutableList.of(node1, node2, node1, node2, node1, node2, node1, node2, node1, node2));
    LifespanScheduler lifespanScheduler = getAffinityLifespanScheduler(bucketNodeMap);
    TestDynamicLifespanScheduler.TestingSourceScheduler sourceScheduler = new TestDynamicLifespanScheduler.TestingSourceScheduler();
    lifespanScheduler.scheduleInitial(sourceScheduler);
    lifespanScheduler.onLifespanExecutionFinished(sourceScheduler.getLastStartedLifespans());
    assertEquals(sourceScheduler.getLastStartedLifespans().size(), 2);
    sourceScheduler.getLastStartedLifespans().clear();
    while (!lifespanScheduler.allLifespanExecutionFinished()) {
        lifespanScheduler.schedule(sourceScheduler);
        lifespanScheduler.onLifespanExecutionFinished(sourceScheduler.getLastStartedLifespans());
        assertEquals(sourceScheduler.getLastStartedLifespans().size(), 2);
        sourceScheduler.getLastStartedLifespans().clear();
    }
}

Example 5 with BucketNodeMap

use of com.facebook.presto.execution.scheduler.BucketNodeMap in project presto by prestodb.

the class PlanFragmenter method analyzeGroupedExecution.

/*
     * In theory, recoverable grouped execution should be decided at query section level (i.e. a connected component of stages connected by remote exchanges).
     * This is because supporting mixed recoverable execution and non-recoverable execution within a query section adds unnecessary complications but provides little benefit,
     * because a single task failure is still likely to fail the non-recoverable stage.
     * However, since the concept of "query section" is not introduced until execution time as of now, it needs significant hacks to decide at fragmenting time.

     * TODO: We should introduce "query section" and make recoverability analysis done at query section level.
     */
private SubPlan analyzeGroupedExecution(Session session, SubPlan subPlan, boolean parentContainsTableFinish) {
    PlanFragment fragment = subPlan.getFragment();
    GroupedExecutionProperties properties = fragment.getRoot().accept(new GroupedExecutionTagger(session, metadata, nodePartitioningManager), null);
    if (properties.isSubTreeUseful()) {
        boolean preferDynamic = fragment.getRemoteSourceNodes().stream().allMatch(node -> node.getExchangeType() == REPLICATE);
        BucketNodeMap bucketNodeMap = nodePartitioningManager.getBucketNodeMap(session, fragment.getPartitioning(), preferDynamic);
        if (bucketNodeMap.isDynamic()) {
            /*
                 * We currently only support recoverable grouped execution if the following statements hold true:
                 *   - Current session enables recoverable grouped execution and table writer merge operator
                 *   - Parent sub plan contains TableFinishNode
                 *   - Current sub plan's root is TableWriterMergeNode or TableWriterNode
                 *   - Input connectors supports split source rewind
                 *   - Output connectors supports partition commit
                 *   - Bucket node map uses dynamic scheduling
                 *   - One table writer per task
                 */
            boolean recoverable = isRecoverableGroupedExecutionEnabled(session) && isTableWriterMergeOperatorEnabled(session) && parentContainsTableFinish && (fragment.getRoot() instanceof TableWriterMergeNode || fragment.getRoot() instanceof TableWriterNode) && properties.isRecoveryEligible();
            if (recoverable) {
                fragment = fragment.withRecoverableGroupedExecution(properties.getCapableTableScanNodes(), properties.getTotalLifespans());
            } else {
                fragment = fragment.withDynamicLifespanScheduleGroupedExecution(properties.getCapableTableScanNodes(), properties.getTotalLifespans());
            }
        } else {
            fragment = fragment.withFixedLifespanScheduleGroupedExecution(properties.getCapableTableScanNodes(), properties.getTotalLifespans());
        }
    }
    ImmutableList.Builder<SubPlan> result = ImmutableList.builder();
    boolean containsTableFinishNode = containsTableFinishNode(fragment);
    for (SubPlan child : subPlan.getChildren()) {
        result.add(analyzeGroupedExecution(session, child, containsTableFinishNode));
    }
    return new SubPlan(fragment, result.build());
}