Examples with TestingTransactionHandle com.facebook.presto.testing.TestingTransactionHandle used on opensource projects

Example 6 with TestingTransactionHandle

use of com.facebook.presto.testing.TestingTransactionHandle in project presto by prestodb.

the class TestNodeScheduler method testMaxSplitsPerNodePerTask.

@Test
public void testMaxSplitsPerNodePerTask() {
    TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
    InternalNode newNode = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, newNode);
    ImmutableList.Builder<Split> initialSplits = ImmutableList.builder();
    for (int i = 0; i < 20; i++) {
        initialSplits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote()));
    }
    List<RemoteTask> tasks = new ArrayList<>();
    MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor);
    for (InternalNode node : nodeManager.getActiveConnectorNodes(CONNECTOR_ID)) {
        // Max out number of splits on node
        TaskId taskId = new TaskId("test", 1, 0, 1);
        RemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, node, initialSplits.build(), nodeTaskMap.createTaskStatsTracker(node, taskId));
        nodeTaskMap.addTask(node, remoteTask);
        tasks.add(remoteTask);
    }
    TaskId taskId = new TaskId("test", 1, 0, 2);
    RemoteTask newRemoteTask = remoteTaskFactory.createTableScanTask(taskId, newNode, initialSplits.build(), nodeTaskMap.createTaskStatsTracker(newNode, taskId));
    // Max out pending splits on new node
    taskMap.put(newNode, newRemoteTask);
    nodeTaskMap.addTask(newNode, newRemoteTask);
    tasks.add(newRemoteTask);
    Set<Split> splits = new HashSet<>();
    for (int i = 0; i < 5; i++) {
        splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote()));
    }
    Multimap<InternalNode, Split> assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments();
    // no split should be assigned to the newNode, as it already has
    // maxSplitsPerNode + maxSplitsPerNodePerTask assigned to it
    // Splits should be scheduled on the other three nodes
    assertEquals(assignments.keySet().size(), 3);
    // No splits scheduled on the maxed out node
    assertFalse(assignments.keySet().contains(newNode));
    for (RemoteTask task : tasks) {
        task.abort();
    }
    assertEquals(nodeTaskMap.getPartitionedSplitsOnNode(newNode), PartitionedSplitsInfo.forZeroSplits());
}

Example 7 with TestingTransactionHandle

use of com.facebook.presto.testing.TestingTransactionHandle in project presto by prestodb.

the class TestNodeScheduler method testMaxSplitsPerNode.

@Test
public void testMaxSplitsPerNode() {
    TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
    InternalNode newNode = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, newNode);
    ImmutableList.Builder<Split> initialSplits = ImmutableList.builder();
    for (int i = 0; i < 10; i++) {
        initialSplits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote()));
    }
    MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor);
    // Max out number of splits on node
    TaskId taskId1 = new TaskId("test", 1, 0, 1);
    RemoteTask remoteTask1 = remoteTaskFactory.createTableScanTask(taskId1, newNode, initialSplits.build(), nodeTaskMap.createTaskStatsTracker(newNode, taskId1));
    nodeTaskMap.addTask(newNode, remoteTask1);
    TaskId taskId2 = new TaskId("test", 1, 0, 2);
    RemoteTask remoteTask2 = remoteTaskFactory.createTableScanTask(taskId2, newNode, initialSplits.build(), nodeTaskMap.createTaskStatsTracker(newNode, taskId2));
    nodeTaskMap.addTask(newNode, remoteTask2);
    Set<Split> splits = new HashSet<>();
    for (int i = 0; i < 5; i++) {
        splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote()));
    }
    Multimap<InternalNode, Split> assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments();
    // no split should be assigned to the newNode, as it already has maxNodeSplits assigned to it
    assertFalse(assignments.keySet().contains(newNode));
    remoteTask1.abort();
    remoteTask2.abort();
    assertEquals(nodeTaskMap.getPartitionedSplitsOnNode(newNode), PartitionedSplitsInfo.forZeroSplits());
}

Example 8 with TestingTransactionHandle

use of com.facebook.presto.testing.TestingTransactionHandle in project presto by prestodb.

the class TestNodeScheduler method testAffinityAssignmentNotSupported.

@Test
public void testAffinityAssignmentNotSupported() {
    NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
    TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
    NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
    LegacyNetworkTopology legacyNetworkTopology = new LegacyNetworkTopology();
    NodeScheduler nodeScheduler = new NodeScheduler(new NetworkLocationCache(legacyNetworkTopology), legacyNetworkTopology, nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, new Duration(0, SECONDS), new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
    NodeSelector nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 2);
    Set<Split> splits = new HashSet<>();
    splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(HostAddress.fromString("127.0.0.1:10"))));
    splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(HostAddress.fromString("127.0.0.1:10"))));
    SplitPlacementResult splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of());
    Set<InternalNode> internalNodes = splitPlacementResult.getAssignments().keySet();
    // Split doesn't support affinity schedule, fall back to random schedule
    assertEquals(internalNodes.size(), 2);
}

Example 9 with TestingTransactionHandle

use of com.facebook.presto.testing.TestingTransactionHandle in project presto by prestodb.

the class TestNodeScheduler method testMaxTasksPerStageWittLimit.

@Test
public void testMaxTasksPerStageWittLimit() {
    NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
    TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
    NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
    NodeSelector nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 2);
    Set<Split> splits = new HashSet<>();
    splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote()));
    SplitPlacementResult splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of());
    Set<InternalNode> internalNodes = splitPlacementResult.getAssignments().keySet();
    assertEquals(internalNodes.size(), 1);
    // adding one more split. Total 2
    splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote()));
    splitPlacementResult = nodeSelector.computeAssignments(splits, getRemoteTableScanTask(splitPlacementResult));
    Set<InternalNode> internalNodesSecondCall = splitPlacementResult.getAssignments().keySet();
    assertEquals(internalNodesSecondCall.size(), 2);
    assertTrue(internalNodesSecondCall.containsAll(internalNodes));
    // adding one more split. Total 3
    splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote()));
    splitPlacementResult = nodeSelector.computeAssignments(splits, getRemoteTableScanTask(splitPlacementResult));
    assertEquals(splitPlacementResult.getAssignments().keySet().size(), 2);
    assertEquals(splitPlacementResult.getAssignments().keySet(), internalNodesSecondCall);
}

Example 10 with TestingTransactionHandle

use of com.facebook.presto.testing.TestingTransactionHandle in project presto by prestodb.

the class TestNodeScheduler method testAffinityAssignmentWithConsistentHashingWithVirtualNodes.

@Test
public void testAffinityAssignmentWithConsistentHashingWithVirtualNodes() {
    NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
    TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
    NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setNodeSelectionHashStrategy(CONSISTENT_HASHING).setMinVirtualNodeCount(5).setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
    NodeSelector nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 3);
    Set<Split> splits = new HashSet<>();
    IntStream.range(0, 10).forEach(i -> splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestAffinitySplitRemote(i))));
    InternalNode node1 = new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false);
    InternalNode node2 = new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false);
    InternalNode node3 = new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false);
    InternalNode node4 = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false);
    // In setup node 1-3 are added to node manager
    // consistent hashing ring for nodes:
    // entry0 (-1907319920): node2
    // entry1 (-1028466245): node3
    // entry2 ( -546736344): node2
    // entry3 ( 1127574531): node3
    // entry4 ( 1166245243): node1
    // entry5 ( 2145381619): node1
    SplitPlacementResult splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of());
    // hashing value for splits:
    // 0: -1962219106 -> entry0 -> node2
    // 1:   145569539 -> entry3 -> node3
    // 2: -1599101205 -> entry1 -> node3
    // 3:  -165119218 -> entry3 -> node3
    // 4:  1142216720 -> entry4 -> node1
    // 5:  1347620135 -> entry5 -> node1
    // 6:  1232195252 -> entry5 -> node1
    // 7:   427886318 -> entry3 -> node3
    // 8:  1469878697 -> entry5 -> node1
    // 9:   296801082 -> entry3 -> node3
    assertEquals(splitPlacementResult.getAssignments().keySet().size(), 3);
    // node1: split 4, 5, 6, 8
    Collection<ConnectorSplit> node1Splits = splitPlacementResult.getAssignments().get(node1).stream().map(Split::getConnectorSplit).collect(toImmutableSet());
    // node2: split 0
    Collection<Object> node2Splits = splitPlacementResult.getAssignments().get(node2).stream().map(Split::getConnectorSplit).collect(toImmutableSet());
    // node3: split 1, 2, 3, 7, 9
    Collection<Object> node3Splits = splitPlacementResult.getAssignments().get(node3).stream().map(Split::getConnectorSplit).collect(toImmutableSet());
    // Scheduling the same splits on the same set of nodes should give the same assignment
    nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 3);
    splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of());
    assertEquals(splitPlacementResult.getAssignments().get(node1).stream().map(Split::getConnectorSplit).collect(toImmutableSet()), node1Splits);
    assertEquals(splitPlacementResult.getAssignments().get(node2).stream().map(Split::getConnectorSplit).collect(toImmutableSet()), node2Splits);
    assertEquals(splitPlacementResult.getAssignments().get(node3).stream().map(Split::getConnectorSplit).collect(toImmutableSet()), node3Splits);
    // Adding node4, consistent hashing ring for nodes:
    // entry0 (-1907319920): node2
    // entry1 (-1616890413): node4
    // entry2 (-1028466245): node3
    // entry3 ( -546736344): node2
    // entry4 ( 1127574531): node3
    // entry5 ( 1166245243): node1
    // entry6 ( 1691928386): node4
    // entry7 ( 2145381619): node1
    nodeManager.addNode(CONNECTOR_ID, node4);
    nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 3);
    splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of());
    // hashing value for splits:
    // 0: -1962219106 -> entry0 -> node2
    // 1:   145569539 -> entry4 -> node3
    // 2: -1599101205 -> entry2 -> node3
    // 3:  -165119218 -> entry4 -> node3
    // 4:  1142216720 -> entry5 -> node1
    // 5:  1347620135 -> entry6 -> node4
    // 6:  1232195252 -> entry6 -> node4
    // 7:   427886318 -> entry4 -> node3
    // 8:  1469878697 -> entry6 -> node4
    // 9:   296801082 -> entry4 -> node3
    assertEquals(splitPlacementResult.getAssignments().keySet().size(), 4);
    assertEquals(splitPlacementResult.getAssignments().get(node1).stream().map(Split::getConnectorSplit).map(ConnectorSplit::getSplitIdentifier).collect(toImmutableSet()), ImmutableSet.of(4));
    assertEquals(splitPlacementResult.getAssignments().get(node2).stream().map(Split::getConnectorSplit).collect(toImmutableSet()), node2Splits);
    assertEquals(splitPlacementResult.getAssignments().get(node3).stream().map(Split::getConnectorSplit).map(ConnectorSplit::getSplitIdentifier).collect(toImmutableSet()), ImmutableSet.of(1, 2, 3, 7, 9));
    assertEquals(splitPlacementResult.getAssignments().get(node4).stream().map(Split::getConnectorSplit).map(ConnectorSplit::getSplitIdentifier).collect(toImmutableSet()), ImmutableSet.of(5, 6, 8));
}