Examples with InternalNode com.facebook.presto.metadata.InternalNode used on opensource projects

Example 41 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestNodeScheduler method testScheduleLocal.

@Test
public void testScheduleLocal() {
    Split split = new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitLocal());
    Set<Split> splits = ImmutableSet.of(split);
    Map.Entry<InternalNode, Split> assignment = Iterables.getOnlyElement(nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments().entries());
    assertEquals(assignment.getKey().getHostAndPort(), split.getPreferredNodes(new ModularHashingNodeProvider(nodeSelector.getAllNodes())).get(0));
    assertEquals(assignment.getValue(), split);
}

Example 42 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestNodeScheduler method testTtlAwareScheduling.

@Test
public void testTtlAwareScheduling() {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    InternalNode node1 = new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false);
    InternalNode node2 = new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false);
    InternalNode node3 = new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false);
    List<InternalNode> nodes = ImmutableList.of(node1, node2, node3);
    nodeManager.addNode(CONNECTOR_ID, nodes);
    NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
    Instant currentInstant = Instant.now();
    NodeTtl ttl1 = new NodeTtl(ImmutableSet.of(new ConfidenceBasedTtlInfo(currentInstant.plus(5, ChronoUnit.MINUTES).getEpochSecond(), 100)));
    NodeTtl ttl2 = new NodeTtl(ImmutableSet.of(new ConfidenceBasedTtlInfo(currentInstant.plus(30, ChronoUnit.MINUTES).getEpochSecond(), 100)));
    NodeTtl ttl3 = new NodeTtl(ImmutableSet.of(new ConfidenceBasedTtlInfo(currentInstant.plus(2, ChronoUnit.HOURS).getEpochSecond(), 100)));
    Map<NodeInfo, NodeTtl> nodeToTtl = ImmutableMap.of(new NodeInfo(node1.getNodeIdentifier(), node1.getHost()), ttl1, new NodeInfo(node2.getNodeIdentifier(), node2.getHost()), ttl2, new NodeInfo(node3.getNodeIdentifier(), node3.getHost()), ttl3);
    ConfidenceBasedNodeTtlFetcherManager nodeTtlFetcherManager = new ConfidenceBasedNodeTtlFetcherManager(nodeManager, new NodeSchedulerConfig(), new NodeTtlFetcherManagerConfig());
    NodeTtlFetcherFactory nodeTtlFetcherFactory = new TestingNodeTtlFetcherFactory(nodeToTtl);
    nodeTtlFetcherManager.addNodeTtlFetcherFactory(nodeTtlFetcherFactory);
    nodeTtlFetcherManager.load(nodeTtlFetcherFactory.getName(), ImmutableMap.of());
    nodeTtlFetcherManager.refreshTtlInfo();
    TestingQueryManager queryManager = new TestingQueryManager();
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, nodeTtlFetcherManager, queryManager, new SimpleTtlNodeSelectorConfig());
    // Query is estimated to take 20 mins and has been executing for 3 mins, i.e, 17 mins left
    // So only node2 and node3 have enough TTL to run additional work
    Session session = sessionWithTtlAwareSchedulingStrategyAndEstimatedExecutionTime(new Duration(20, TimeUnit.MINUTES));
    NodeSelector nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID);
    queryManager.setExecutionTime(new Duration(3, TimeUnit.MINUTES));
    assertEquals(ImmutableSet.copyOf(nodeSelector.selectRandomNodes(3)), ImmutableSet.of(node2, node3));
    // Query is estimated to take 1 hour and has been executing for 45 mins, i.e, 15 mins left
    // So only node2 and node3 have enough TTL to work on new splits
    session = sessionWithTtlAwareSchedulingStrategyAndEstimatedExecutionTime(new Duration(1, TimeUnit.HOURS));
    nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID);
    queryManager.setExecutionTime(new Duration(45, TimeUnit.MINUTES));
    Set<Split> splits = new HashSet<>();
    for (int i = 0; i < 2; i++) {
        splits.add(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote()));
    }
    Multimap<InternalNode, Split> assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments();
    assertEquals(assignments.size(), 2);
    assertTrue(assignments.keySet().contains(node2));
    assertTrue(assignments.keySet().contains(node3));
    // Query is estimated to take 1 hour and has been executing for 20 mins, i.e, 40 mins left
    // So only node3 has enough TTL to work on new splits
    MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor);
    TaskId taskId = new TaskId("test", 1, 0, 1);
    RemoteTask newRemoteTask = remoteTaskFactory.createTableScanTask(taskId, node2, ImmutableList.of(), nodeTaskMap.createTaskStatsTracker(node2, taskId));
    taskMap.put(node2, newRemoteTask);
    nodeTaskMap.addTask(node2, newRemoteTask);
    session = sessionWithTtlAwareSchedulingStrategyAndEstimatedExecutionTime(new Duration(1, TimeUnit.HOURS));
    nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID);
    queryManager.setExecutionTime(new Duration(20, TimeUnit.MINUTES));
    splits.clear();
    for (int i = 0; i < 2; i++) {
        splits.add(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote()));
    }
    assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments();
    assertEquals(assignments.size(), 2);
    assertEquals(assignments.keySet().size(), 1);
    assertTrue(assignments.keySet().contains(node3));
}

Example 43 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestNodeScheduler method testMaxSplitsPerNodePerTask.

@Test
public void testMaxSplitsPerNodePerTask() {
    TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
    InternalNode newNode = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, newNode);
    ImmutableList.Builder<Split> initialSplits = ImmutableList.builder();
    for (int i = 0; i < 20; i++) {
        initialSplits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote()));
    }
    List<RemoteTask> tasks = new ArrayList<>();
    MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor);
    for (InternalNode node : nodeManager.getActiveConnectorNodes(CONNECTOR_ID)) {
        // Max out number of splits on node
        TaskId taskId = new TaskId("test", 1, 0, 1);
        RemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, node, initialSplits.build(), nodeTaskMap.createTaskStatsTracker(node, taskId));
        nodeTaskMap.addTask(node, remoteTask);
        tasks.add(remoteTask);
    }
    TaskId taskId = new TaskId("test", 1, 0, 2);
    RemoteTask newRemoteTask = remoteTaskFactory.createTableScanTask(taskId, newNode, initialSplits.build(), nodeTaskMap.createTaskStatsTracker(newNode, taskId));
    // Max out pending splits on new node
    taskMap.put(newNode, newRemoteTask);
    nodeTaskMap.addTask(newNode, newRemoteTask);
    tasks.add(newRemoteTask);
    Set<Split> splits = new HashSet<>();
    for (int i = 0; i < 5; i++) {
        splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote()));
    }
    Multimap<InternalNode, Split> assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments();
    // no split should be assigned to the newNode, as it already has
    // maxSplitsPerNode + maxSplitsPerNodePerTask assigned to it
    // Splits should be scheduled on the other three nodes
    assertEquals(assignments.keySet().size(), 3);
    // No splits scheduled on the maxed out node
    assertFalse(assignments.keySet().contains(newNode));
    for (RemoteTask task : tasks) {
        task.abort();
    }
    assertEquals(nodeTaskMap.getPartitionedSplitsOnNode(newNode), PartitionedSplitsInfo.forZeroSplits());
}

Example 44 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestNodeScheduler method testMaxSplitsPerNode.

@Test
public void testMaxSplitsPerNode() {
    TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
    InternalNode newNode = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, newNode);
    ImmutableList.Builder<Split> initialSplits = ImmutableList.builder();
    for (int i = 0; i < 10; i++) {
        initialSplits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote()));
    }
    MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor);
    // Max out number of splits on node
    TaskId taskId1 = new TaskId("test", 1, 0, 1);
    RemoteTask remoteTask1 = remoteTaskFactory.createTableScanTask(taskId1, newNode, initialSplits.build(), nodeTaskMap.createTaskStatsTracker(newNode, taskId1));
    nodeTaskMap.addTask(newNode, remoteTask1);
    TaskId taskId2 = new TaskId("test", 1, 0, 2);
    RemoteTask remoteTask2 = remoteTaskFactory.createTableScanTask(taskId2, newNode, initialSplits.build(), nodeTaskMap.createTaskStatsTracker(newNode, taskId2));
    nodeTaskMap.addTask(newNode, remoteTask2);
    Set<Split> splits = new HashSet<>();
    for (int i = 0; i < 5; i++) {
        splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote()));
    }
    Multimap<InternalNode, Split> assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments();
    // no split should be assigned to the newNode, as it already has maxNodeSplits assigned to it
    assertFalse(assignments.keySet().contains(newNode));
    remoteTask1.abort();
    remoteTask2.abort();
    assertEquals(nodeTaskMap.getPartitionedSplitsOnNode(newNode), PartitionedSplitsInfo.forZeroSplits());
}

Example 45 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestNodeScheduler method testAffinityAssignmentNotSupported.

@Test
public void testAffinityAssignmentNotSupported() {
    NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
    TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
    NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
    LegacyNetworkTopology legacyNetworkTopology = new LegacyNetworkTopology();
    NodeScheduler nodeScheduler = new NodeScheduler(new NetworkLocationCache(legacyNetworkTopology), legacyNetworkTopology, nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, new Duration(0, SECONDS), new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
    NodeSelector nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 2);
    Set<Split> splits = new HashSet<>();
    splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(HostAddress.fromString("127.0.0.1:10"))));
    splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(HostAddress.fromString("127.0.0.1:10"))));
    SplitPlacementResult splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of());
    Set<InternalNode> internalNodes = splitPlacementResult.getAssignments().keySet();
    // Split doesn't support affinity schedule, fall back to random schedule
    assertEquals(internalNodes.size(), 2);
}