Search in sources :

Example 11 with TestingTransactionHandle

use of com.facebook.presto.testing.TestingTransactionHandle in project presto by prestodb.

the class TestNodeScheduler method testMaxUnacknowledgedSplitsPerTask.

@Test
public void testMaxUnacknowledgedSplitsPerTask() {
    int maxUnacknowledgedSplitsPerTask = 5;
    nodeSelector = nodeScheduler.createNodeSelector(sessionWithMaxUnacknowledgedSplitsPerTask(maxUnacknowledgedSplitsPerTask), CONNECTOR_ID, Integer.MAX_VALUE);
    TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
    ImmutableList.Builder<Split> initialSplits = ImmutableList.builder();
    for (int i = 0; i < maxUnacknowledgedSplitsPerTask; i++) {
        initialSplits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote()));
    }
    List<InternalNode> nodes = new ArrayList<>();
    List<MockRemoteTaskFactory.MockRemoteTask> tasks = new ArrayList<>();
    MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor);
    int counter = 1;
    for (InternalNode node : nodeManager.getActiveConnectorNodes(CONNECTOR_ID)) {
        // Max out number of unacknowledged splits on each task
        TaskId taskId = new TaskId("test", 1, 0, counter);
        counter++;
        MockRemoteTaskFactory.MockRemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, node, initialSplits.build(), nodeTaskMap.createTaskStatsTracker(node, taskId));
        nodeTaskMap.addTask(node, remoteTask);
        remoteTask.setMaxUnacknowledgedSplits(maxUnacknowledgedSplitsPerTask);
        remoteTask.setUnacknowledgedSplits(maxUnacknowledgedSplitsPerTask);
        nodes.add(node);
        tasks.add(remoteTask);
    }
    // One split per node
    Set<Split> splits = new HashSet<>();
    for (int i = 0; i < nodes.size(); i++) {
        splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote()));
    }
    SplitPlacementResult splitPlacements = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(tasks));
    // No splits should have been placed, max unacknowledged was already reached
    assertEquals(splitPlacements.getAssignments().size(), 0);
    assertFalse(splitPlacements.getBlocked().isDone());
    // Unblock one task
    MockRemoteTaskFactory.MockRemoteTask taskOne = tasks.get(0);
    taskOne.finishSplits(1);
    taskOne.setUnacknowledgedSplits(taskOne.getUnacknowledgedPartitionedSplitCount() - 1);
    assertTrue(splitPlacements.getBlocked().isDone());
    // Attempt to schedule again, only the node with the unblocked task should be chosen
    splitPlacements = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(tasks));
    assertEquals(splitPlacements.getAssignments().size(), 1);
    assertTrue(splitPlacements.getAssignments().keySet().contains(nodes.get(0)));
    // Make the first node appear to have no splits, unacknowledged splits alone should force the splits to be spread across nodes
    taskOne.clearSplits();
    // Give all tasks with room for 1 unacknowledged split
    tasks.forEach(task -> task.setUnacknowledgedSplits(maxUnacknowledgedSplitsPerTask - 1));
    splitPlacements = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(tasks));
    // One split placed on each node
    assertEquals(splitPlacements.getAssignments().size(), nodes.size());
    assertTrue(splitPlacements.getAssignments().keySet().containsAll(nodes));
}
Also used : ImmutableList(com.google.common.collect.ImmutableList) ArrayList(java.util.ArrayList) TestingTransactionHandle(com.facebook.presto.testing.TestingTransactionHandle) InternalNode(com.facebook.presto.metadata.InternalNode) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) Split(com.facebook.presto.metadata.Split) SplitPlacementResult(com.facebook.presto.execution.scheduler.SplitPlacementResult) HashSet(java.util.HashSet) Test(org.testng.annotations.Test)

Example 12 with TestingTransactionHandle

use of com.facebook.presto.testing.TestingTransactionHandle in project presto by prestodb.

the class TestNodeScheduler method testHardAffinityAssignment.

@Test
public void testHardAffinityAssignment() {
    NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
    TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
    NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
    NodeSelector nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 3);
    Set<Split> splits = new HashSet<>();
    // Adding one more split (1 % 3 = 1), 1 splits will be distributed to 1 nodes
    splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestHardAffinitySplitRemote()));
    splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestHardAffinitySplitRemote()));
    splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestHardAffinitySplitRemote()));
    SplitPlacementResult splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of());
    for (Split split : splitPlacementResult.getAssignments().values()) {
        assertTrue(split.getSplitContext().isCacheable());
    }
}
Also used : NodeSchedulerConfig(com.facebook.presto.execution.scheduler.NodeSchedulerConfig) ThrowingNodeTtlFetcherManager(com.facebook.presto.ttl.nodettlfetchermanagers.ThrowingNodeTtlFetcherManager) NoOpQueryManager(com.facebook.presto.dispatcher.NoOpQueryManager) NodeSelectionStats(com.facebook.presto.execution.scheduler.nodeSelection.NodeSelectionStats) LegacyNetworkTopology(com.facebook.presto.execution.scheduler.LegacyNetworkTopology) NodeScheduler(com.facebook.presto.execution.scheduler.NodeScheduler) TestingTransactionHandle(com.facebook.presto.testing.TestingTransactionHandle) NodeSelector(com.facebook.presto.execution.scheduler.nodeSelection.NodeSelector) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) Split(com.facebook.presto.metadata.Split) SplitPlacementResult(com.facebook.presto.execution.scheduler.SplitPlacementResult) SimpleTtlNodeSelectorConfig(com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig) HashSet(java.util.HashSet) Test(org.testng.annotations.Test)

Aggregations

Split (com.facebook.presto.metadata.Split)12 ConnectorSplit (com.facebook.presto.spi.ConnectorSplit)12 TestingTransactionHandle (com.facebook.presto.testing.TestingTransactionHandle)12 Test (org.testng.annotations.Test)12 InternalNode (com.facebook.presto.metadata.InternalNode)11 HashSet (java.util.HashSet)11 SplitPlacementResult (com.facebook.presto.execution.scheduler.SplitPlacementResult)9 NoOpQueryManager (com.facebook.presto.dispatcher.NoOpQueryManager)8 NodeScheduler (com.facebook.presto.execution.scheduler.NodeScheduler)8 NodeSchedulerConfig (com.facebook.presto.execution.scheduler.NodeSchedulerConfig)8 NodeSelectionStats (com.facebook.presto.execution.scheduler.nodeSelection.NodeSelectionStats)8 NodeSelector (com.facebook.presto.execution.scheduler.nodeSelection.NodeSelector)8 SimpleTtlNodeSelectorConfig (com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig)8 ThrowingNodeTtlFetcherManager (com.facebook.presto.ttl.nodettlfetchermanagers.ThrowingNodeTtlFetcherManager)8 LegacyNetworkTopology (com.facebook.presto.execution.scheduler.LegacyNetworkTopology)7 ImmutableList (com.google.common.collect.ImmutableList)4 NetworkLocationCache (com.facebook.presto.execution.scheduler.NetworkLocationCache)3 InMemoryNodeManager (com.facebook.presto.metadata.InMemoryNodeManager)2 PlanNodeId (com.facebook.presto.spi.plan.PlanNodeId)2 ImmutableSet (com.google.common.collect.ImmutableSet)2