Examples with PartitionedSplitsInfo com.facebook.presto.execution.PartitionedSplitsInfo used on opensource projects

Example 1 with PartitionedSplitsInfo

use of com.facebook.presto.execution.PartitionedSplitsInfo in project presto by prestodb.

the class TestSourcePartitionedScheduler method testBlockCausesFullSchedule.

@Test
public void testBlockCausesFullSchedule() {
    NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
    // Schedule 60 splits - filling up all nodes
    SubPlan firstPlan = createPlan();
    SqlStageExecution firstStage = createSqlStageExecution(firstPlan, nodeTaskMap);
    StageScheduler firstScheduler = getSourcePartitionedScheduler(createFixedSplitSource(60, TestingSplit::createRemoteSplit), firstStage, nodeManager, nodeTaskMap, 200);
    ScheduleResult scheduleResult = firstScheduler.schedule();
    assertEffectivelyFinished(scheduleResult, firstScheduler);
    assertTrue(scheduleResult.getBlocked().isDone());
    assertEquals(scheduleResult.getNewTasks().size(), 3);
    assertEquals(firstStage.getAllTasks().size(), 3);
    for (RemoteTask remoteTask : firstStage.getAllTasks()) {
        PartitionedSplitsInfo splitsInfo = remoteTask.getPartitionedSplitsInfo();
        assertEquals(splitsInfo.getCount(), 20);
    }
    // Schedule more splits in another query, which will block since all nodes are full
    SubPlan secondPlan = createPlan();
    SqlStageExecution secondStage = createSqlStageExecution(secondPlan, nodeTaskMap);
    StageScheduler secondScheduler = getSourcePartitionedScheduler(createFixedSplitSource(5, TestingSplit::createRemoteSplit), secondStage, nodeManager, nodeTaskMap, 200);
    scheduleResult = secondScheduler.schedule();
    assertFalse(scheduleResult.isFinished());
    assertTrue(scheduleResult.getBlocked().isDone());
    assertEquals(scheduleResult.getNewTasks().size(), 3);
    assertEquals(secondStage.getAllTasks().size(), 3);
    for (RemoteTask remoteTask : secondStage.getAllTasks()) {
        PartitionedSplitsInfo splitsInfo = remoteTask.getPartitionedSplitsInfo();
        assertEquals(splitsInfo.getCount(), 0);
    }
    firstStage.abort();
    secondStage.abort();
}

Example 2 with PartitionedSplitsInfo

use of com.facebook.presto.execution.PartitionedSplitsInfo in project presto by prestodb.

the class TestSourcePartitionedScheduler method testBalancedSplitAssignment.

@Test
public void testBalancedSplitAssignment() {
    // use private node manager so we can add a node later
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(CONNECTOR_ID, new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false), new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false), new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false));
    NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
    // Schedule 15 splits - there are 3 nodes, each node should get 5 splits
    SubPlan firstPlan = createPlan();
    SqlStageExecution firstStage = createSqlStageExecution(firstPlan, nodeTaskMap);
    StageScheduler firstScheduler = getSourcePartitionedScheduler(createFixedSplitSource(15, TestingSplit::createRemoteSplit), firstStage, nodeManager, nodeTaskMap, 200);
    ScheduleResult scheduleResult = firstScheduler.schedule();
    assertEffectivelyFinished(scheduleResult, firstScheduler);
    assertTrue(scheduleResult.getBlocked().isDone());
    assertEquals(scheduleResult.getNewTasks().size(), 3);
    assertEquals(firstStage.getAllTasks().size(), 3);
    for (RemoteTask remoteTask : firstStage.getAllTasks()) {
        PartitionedSplitsInfo splitsInfo = remoteTask.getPartitionedSplitsInfo();
        assertEquals(splitsInfo.getCount(), 5);
    }
    // Add new node
    InternalNode additionalNode = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, additionalNode);
    // Schedule 5 splits in another query. Since the new node does not have any splits, all 5 splits are assigned to the new node
    SubPlan secondPlan = createPlan();
    SqlStageExecution secondStage = createSqlStageExecution(secondPlan, nodeTaskMap);
    StageScheduler secondScheduler = getSourcePartitionedScheduler(createFixedSplitSource(5, TestingSplit::createRemoteSplit), secondStage, nodeManager, nodeTaskMap, 200);
    scheduleResult = secondScheduler.schedule();
    assertEffectivelyFinished(scheduleResult, secondScheduler);
    assertTrue(scheduleResult.getBlocked().isDone());
    assertEquals(scheduleResult.getNewTasks().size(), 1);
    assertEquals(secondStage.getAllTasks().size(), 1);
    RemoteTask task = secondStage.getAllTasks().get(0);
    assertEquals(task.getPartitionedSplitsInfo().getCount(), 5);
    firstStage.abort();
    secondStage.abort();
}

Example 3 with PartitionedSplitsInfo

use of com.facebook.presto.execution.PartitionedSplitsInfo in project presto by prestodb.

the class HttpRemoteTask method getQueuedPartitionedSplitsInfo.

@Override
public PartitionedSplitsInfo getQueuedPartitionedSplitsInfo() {
    TaskStatus taskStatus = getTaskStatus();
    if (taskStatus.getState().isDone()) {
        return PartitionedSplitsInfo.forZeroSplits();
    }
    PartitionedSplitsInfo unacknowledgedSplitsInfo = getUnacknowledgedPartitionedSplitsInfo();
    int count = unacknowledgedSplitsInfo.getCount() + taskStatus.getQueuedPartitionedDrivers();
    long weight = unacknowledgedSplitsInfo.getWeightSum() + taskStatus.getQueuedPartitionedSplitsWeight();
    return PartitionedSplitsInfo.forSplitCountAndWeightSum(count, weight);
}

Example 4 with PartitionedSplitsInfo

use of com.facebook.presto.execution.PartitionedSplitsInfo in project presto by prestodb.

the class NodeAssignmentStats method getTotalSplitsWeight.

public long getTotalSplitsWeight(InternalNode node) {
    PartitionedSplitsInfo nodeTotalSplits = nodeTotalSplitsInfo.computeIfAbsent(node, nodeTaskMap::getPartitionedSplitsOnNode);
    PendingSplitInfo stageInfo = stageQueuedSplitInfo.get(node.getNodeIdentifier());
    if (stageInfo == null) {
        return nodeTotalSplits.getWeightSum();
    }
    return addExact(nodeTotalSplits.getWeightSum(), stageInfo.getAssignedSplitsWeight());
}

Example 5 with PartitionedSplitsInfo

use of com.facebook.presto.execution.PartitionedSplitsInfo in project presto by prestodb.

the class HttpRemoteTask method getPartitionedSplitsInfo.

@Override
public PartitionedSplitsInfo getPartitionedSplitsInfo() {
    TaskStatus taskStatus = getTaskStatus();
    if (taskStatus.getState().isDone()) {
        return PartitionedSplitsInfo.forZeroSplits();
    }
    PartitionedSplitsInfo unacknowledgedSplitsInfo = getUnacknowledgedPartitionedSplitsInfo();
    int count = unacknowledgedSplitsInfo.getCount() + taskStatus.getQueuedPartitionedDrivers() + taskStatus.getRunningPartitionedDrivers();
    long weight = unacknowledgedSplitsInfo.getWeightSum() + taskStatus.getQueuedPartitionedSplitsWeight() + taskStatus.getRunningPartitionedSplitsWeight();
    return PartitionedSplitsInfo.forSplitCountAndWeightSum(count, weight);
}