Search in sources :

Example 6 with PartitionedSplitsInfo

use of com.facebook.presto.execution.PartitionedSplitsInfo in project presto by prestodb.

the class TestSourcePartitionedScheduler method testScheduleSplitsBlock.

@Test
public void testScheduleSplitsBlock() {
    SubPlan plan = createPlan();
    NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
    SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap);
    StageScheduler scheduler = getSourcePartitionedScheduler(createFixedSplitSource(80, TestingSplit::createRemoteSplit), stage, nodeManager, nodeTaskMap, 1);
    // schedule first 60 splits, which will cause the scheduler to block
    for (int i = 0; i <= 60; i++) {
        ScheduleResult scheduleResult = scheduler.schedule();
        assertFalse(scheduleResult.isFinished());
        // blocks at 20 per node
        assertEquals(scheduleResult.getBlocked().isDone(), i != 60);
        // first three splits create new tasks
        assertEquals(scheduleResult.getNewTasks().size(), i < 3 ? 1 : 0);
        assertEquals(stage.getAllTasks().size(), i < 3 ? i + 1 : 3);
        assertPartitionedSplitCount(stage, min(i + 1, 60));
    }
    for (RemoteTask remoteTask : stage.getAllTasks()) {
        PartitionedSplitsInfo splitsInfo = remoteTask.getPartitionedSplitsInfo();
        assertEquals(splitsInfo.getCount(), 20);
    }
    // todo rewrite MockRemoteTask to fire a tate transition when splits are cleared, and then validate blocked future completes
    // drop the 20 splits from one node
    ((MockRemoteTask) stage.getAllTasks().get(0)).clearSplits();
    // schedule remaining 20 splits
    for (int i = 0; i < 20; i++) {
        ScheduleResult scheduleResult = scheduler.schedule();
        // finishes when last split is fetched
        if (i == 19) {
            assertEffectivelyFinished(scheduleResult, scheduler);
        } else {
            assertFalse(scheduleResult.isFinished());
        }
        // does not block again
        assertTrue(scheduleResult.getBlocked().isDone());
        // no additional tasks will be created
        assertEquals(scheduleResult.getNewTasks().size(), 0);
        assertEquals(stage.getAllTasks().size(), 3);
        // we dropped 20 splits so start at 40 and count to 60
        assertPartitionedSplitCount(stage, min(i + 41, 60));
    }
    for (RemoteTask remoteTask : stage.getAllTasks()) {
        PartitionedSplitsInfo splitsInfo = remoteTask.getPartitionedSplitsInfo();
        assertEquals(splitsInfo.getCount(), 20);
    }
    stage.abort();
}
Also used : NodeTaskMap(com.facebook.presto.execution.NodeTaskMap) PartitionedSplitsInfo(com.facebook.presto.execution.PartitionedSplitsInfo) MockRemoteTask(com.facebook.presto.execution.MockRemoteTaskFactory.MockRemoteTask) MockRemoteTask(com.facebook.presto.execution.MockRemoteTaskFactory.MockRemoteTask) RemoteTask(com.facebook.presto.execution.RemoteTask) SubPlan(com.facebook.presto.sql.planner.SubPlan) SqlStageExecution(com.facebook.presto.execution.SqlStageExecution) SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler(com.facebook.presto.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler) Test(org.testng.annotations.Test)

Example 7 with PartitionedSplitsInfo

use of com.facebook.presto.execution.PartitionedSplitsInfo in project presto by prestodb.

the class TestSourcePartitionedScheduler method testScheduleSplitsBatched.

@Test
public void testScheduleSplitsBatched() {
    SubPlan plan = createPlan();
    NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
    SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap);
    StageScheduler scheduler = getSourcePartitionedScheduler(createFixedSplitSource(60, TestingSplit::createRemoteSplit), stage, nodeManager, nodeTaskMap, 7);
    for (int i = 0; i <= (60 / 7); i++) {
        ScheduleResult scheduleResult = scheduler.schedule();
        // finishes when last split is fetched
        if (i == (60 / 7)) {
            assertEffectivelyFinished(scheduleResult, scheduler);
        } else {
            assertFalse(scheduleResult.isFinished());
        }
        // never blocks
        assertTrue(scheduleResult.getBlocked().isDone());
        // first three splits create new tasks
        assertEquals(scheduleResult.getNewTasks().size(), i == 0 ? 3 : 0);
        assertEquals(stage.getAllTasks().size(), 3);
        assertPartitionedSplitCount(stage, min((i + 1) * 7, 60));
    }
    for (RemoteTask remoteTask : stage.getAllTasks()) {
        PartitionedSplitsInfo splitsInfo = remoteTask.getPartitionedSplitsInfo();
        assertEquals(splitsInfo.getCount(), 20);
    }
    stage.abort();
}
Also used : NodeTaskMap(com.facebook.presto.execution.NodeTaskMap) PartitionedSplitsInfo(com.facebook.presto.execution.PartitionedSplitsInfo) MockRemoteTask(com.facebook.presto.execution.MockRemoteTaskFactory.MockRemoteTask) RemoteTask(com.facebook.presto.execution.RemoteTask) SubPlan(com.facebook.presto.sql.planner.SubPlan) SqlStageExecution(com.facebook.presto.execution.SqlStageExecution) SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler(com.facebook.presto.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler) Test(org.testng.annotations.Test)

Example 8 with PartitionedSplitsInfo

use of com.facebook.presto.execution.PartitionedSplitsInfo in project presto by prestodb.

the class TestSourcePartitionedScheduler method testScheduleSplitsOneAtATime.

@Test
public void testScheduleSplitsOneAtATime() {
    SubPlan plan = createPlan();
    NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
    SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap);
    StageScheduler scheduler = getSourcePartitionedScheduler(createFixedSplitSource(60, TestingSplit::createRemoteSplit), stage, nodeManager, nodeTaskMap, 1);
    for (int i = 0; i < 60; i++) {
        ScheduleResult scheduleResult = scheduler.schedule();
        // only finishes when last split is fetched
        if (i == 59) {
            assertEffectivelyFinished(scheduleResult, scheduler);
        } else {
            assertFalse(scheduleResult.isFinished());
        }
        // never blocks
        assertTrue(scheduleResult.getBlocked().isDone());
        // first three splits create new tasks
        assertEquals(scheduleResult.getNewTasks().size(), i < 3 ? 1 : 0);
        assertEquals(stage.getAllTasks().size(), i < 3 ? i + 1 : 3);
        assertPartitionedSplitCount(stage, min(i + 1, 60));
    }
    for (RemoteTask remoteTask : stage.getAllTasks()) {
        PartitionedSplitsInfo splitsInfo = remoteTask.getPartitionedSplitsInfo();
        assertEquals(splitsInfo.getCount(), 20);
    }
    stage.abort();
}
Also used : NodeTaskMap(com.facebook.presto.execution.NodeTaskMap) PartitionedSplitsInfo(com.facebook.presto.execution.PartitionedSplitsInfo) MockRemoteTask(com.facebook.presto.execution.MockRemoteTaskFactory.MockRemoteTask) RemoteTask(com.facebook.presto.execution.RemoteTask) SubPlan(com.facebook.presto.sql.planner.SubPlan) SqlStageExecution(com.facebook.presto.execution.SqlStageExecution) SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler(com.facebook.presto.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler) Test(org.testng.annotations.Test)

Aggregations

PartitionedSplitsInfo (com.facebook.presto.execution.PartitionedSplitsInfo)8 MockRemoteTask (com.facebook.presto.execution.MockRemoteTaskFactory.MockRemoteTask)5 NodeTaskMap (com.facebook.presto.execution.NodeTaskMap)5 RemoteTask (com.facebook.presto.execution.RemoteTask)5 SqlStageExecution (com.facebook.presto.execution.SqlStageExecution)5 SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler (com.facebook.presto.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler)5 SubPlan (com.facebook.presto.sql.planner.SubPlan)5 Test (org.testng.annotations.Test)5 TaskStatus (com.facebook.presto.execution.TaskStatus)2 InMemoryNodeManager (com.facebook.presto.metadata.InMemoryNodeManager)1 InternalNode (com.facebook.presto.metadata.InternalNode)1