Examples with InternalNode com.facebook.presto.metadata.InternalNode used on opensource projects

Example 21 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestNodeScheduler method testCpuUsage.

@Test
public void testCpuUsage() {
    MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor);
    InternalNode chosenNode = Iterables.get(nodeManager.getActiveConnectorNodes(CONNECTOR_ID), 0);
    TaskId taskId1 = new TaskId("test", 1, 0, 1);
    List<Split> splits = ImmutableList.of(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote()), new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote()));
    RemoteTask remoteTask1 = remoteTaskFactory.createTableScanTask(taskId1, chosenNode, splits, nodeTaskMap.createTaskStatsTracker(chosenNode, taskId1));
    TaskId taskId2 = new TaskId("test", 1, 0, 2);
    RemoteTask remoteTask2 = remoteTaskFactory.createTableScanTask(taskId2, chosenNode, ImmutableList.of(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote())), nodeTaskMap.createTaskStatsTracker(chosenNode, taskId2));
    nodeTaskMap.addTask(chosenNode, remoteTask1);
    nodeTaskMap.addTask(chosenNode, remoteTask2);
    remoteTask2.addSplits(ImmutableMultimap.<PlanNodeId, Split>builder().putAll(new PlanNodeId("sourceId"), splits).build());
    assertEquals(nodeTaskMap.getNodeCpuUtilizationPercentage(chosenNode), 100D);
    remoteTask1.addSplits(ImmutableMultimap.<PlanNodeId, Split>builder().putAll(new PlanNodeId("sourceId"), splits).build());
    assertEquals(nodeTaskMap.getNodeCpuUtilizationPercentage(chosenNode), 200D);
    remoteTask1.abort();
    assertEquals(nodeTaskMap.getNodeCpuUtilizationPercentage(chosenNode), 100D);
    remoteTask2.abort();
    assertEquals(nodeTaskMap.getNodeCpuUtilizationPercentage(chosenNode), 0D);
}

Example 22 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestNodeScheduler method testMoreSplitsAssignedWhenSplitsWeightsAreSmall.

@Test
public void testMoreSplitsAssignedWhenSplitsWeightsAreSmall() {
    int standardSplitsPerNode = nodeSchedulerConfig.getMaxSplitsPerNode();
    int standardPendingSplitsPerTask = nodeSchedulerConfig.getMaxPendingSplitsPerTask();
    int fullyLoadedStandardSplitCount = standardSplitsPerNode + standardPendingSplitsPerTask;
    long weightLimitPerNode = SplitWeight.rawValueForStandardSplitCount(standardSplitsPerNode);
    long weightLimitPendingPerTask = SplitWeight.rawValueForStandardSplitCount(standardPendingSplitsPerTask);
    long fullyLoadedStandardSplitWeight = weightLimitPerNode + weightLimitPendingPerTask;
    // Single worker node
    nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 1);
    InternalNode workerNode = nodeSelector.selectRandomNodes(1).get(0);
    MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor);
    TaskId taskId = new TaskId("test", 1, 0, 1);
    MockRemoteTaskFactory.MockRemoteTask task = remoteTaskFactory.createTableScanTask(taskId, workerNode, ImmutableList.of(), nodeTaskMap.createTaskStatsTracker(workerNode, taskId));
    TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
    ImmutableSet.Builder<Split> splitsBuilder = ImmutableSet.builderWithExpectedSize(fullyLoadedStandardSplitCount * 2);
    // Create 2x more splits than the standard split count limit, at 1/2 the standard weight
    SplitWeight halfWeight = SplitWeight.fromProportion(0.5);
    for (int i = 0; i < fullyLoadedStandardSplitCount * 2; i++) {
        splitsBuilder.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(halfWeight)));
    }
    Set<Split> splits = splitsBuilder.build();
    // Verify we arrived at the exact weight limit
    assertEquals(SplitWeight.rawValueSum(splits, Split::getSplitWeight), fullyLoadedStandardSplitWeight);
    // Node assignment limit met
    SplitPlacementResult result = nodeSelector.computeAssignments(splits, ImmutableList.of(task));
    assertEquals(result.getAssignments().get(workerNode).size(), standardSplitsPerNode * 2);
    assertEquals(SplitWeight.rawValueSum(result.getAssignments().get(workerNode), Split::getSplitWeight), weightLimitPerNode);
    // Mark all splits as running
    task.addSplits(ImmutableMultimap.<PlanNodeId, Split>builder().putAll(new PlanNodeId("sourceId"), result.getAssignments().get(workerNode)).build());
    task.startSplits(result.getAssignments().get(workerNode).size());
    // Per task pending splits limit met
    Set<Split> remainingSplits = Sets.difference(splits, ImmutableSet.copyOf(result.getAssignments().get(workerNode)));
    SplitPlacementResult secondResults = nodeSelector.computeAssignments(remainingSplits, ImmutableList.of(task));
    assertEquals(secondResults.getAssignments().get(workerNode).size(), standardPendingSplitsPerTask * 2);
    assertEquals(SplitWeight.rawValueSum(secondResults.getAssignments().get(workerNode), Split::getSplitWeight), weightLimitPendingPerTask);
    task.addSplits(ImmutableMultimap.<PlanNodeId, Split>builder().putAll(new PlanNodeId("sourceId"), secondResults.getAssignments().get(workerNode)).build());
    assertEquals(nodeTaskMap.getPartitionedSplitsOnNode(workerNode), // 2x fully loaded standard count, full weight limit reached
    PartitionedSplitsInfo.forSplitCountAndWeightSum(fullyLoadedStandardSplitCount * 2, fullyLoadedStandardSplitWeight));
    // No more splits assigned when full
    SplitPlacementResult resultWhenFull = nodeSelector.computeAssignments(ImmutableSet.of(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote())), ImmutableList.of(task));
    assertTrue(resultWhenFull.getAssignments().isEmpty());
}

Example 23 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestNodeScheduler method testTaskCompletion.

@Test
public void testTaskCompletion() throws Exception {
    MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor);
    InternalNode chosenNode = Iterables.get(nodeManager.getActiveConnectorNodes(CONNECTOR_ID), 0);
    TaskId taskId = new TaskId("test", 1, 0, 1);
    RemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, chosenNode, ImmutableList.of(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote())), nodeTaskMap.createTaskStatsTracker(chosenNode, taskId));
    nodeTaskMap.addTask(chosenNode, remoteTask);
    assertEquals(nodeTaskMap.getPartitionedSplitsOnNode(chosenNode), standardWeightSplitsInfo(1));
    remoteTask.abort();
    // Sleep until cache expires
    MILLISECONDS.sleep(100);
    assertEquals(nodeTaskMap.getPartitionedSplitsOnNode(chosenNode), PartitionedSplitsInfo.forZeroSplits());
    remoteTask.abort();
    assertEquals(nodeTaskMap.getPartitionedSplitsOnNode(chosenNode), PartitionedSplitsInfo.forZeroSplits());
}

Example 24 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class ConsistentHashingNodeProvider method get.

@Override
public List<HostAddress> get(String key, int count) {
    if (count > nodeCount) {
        count = nodeCount;
    }
    ImmutableList.Builder<HostAddress> nodes = ImmutableList.builder();
    Set<HostAddress> unique = new HashSet<>();
    int hashKey = HASH_FUNCTION.hashString(format("%s", key), UTF_8).asInt();
    Map.Entry<Integer, InternalNode> entry = candidates.ceilingEntry(hashKey);
    HostAddress candidate;
    SortedMap<Integer, InternalNode> nextEntries;
    if (entry != null) {
        candidate = entry.getValue().getHostAndPort();
        nextEntries = candidates.tailMap(entry.getKey(), false);
    } else {
        candidate = candidates.firstEntry().getValue().getHostAndPort();
        nextEntries = candidates.tailMap(candidates.firstKey(), false);
    }
    unique.add(candidate);
    nodes.add(candidate);
    while (unique.size() < count) {
        for (Map.Entry<Integer, InternalNode> next : nextEntries.entrySet()) {
            candidate = next.getValue().getHostAndPort();
            if (!unique.contains(candidate)) {
                unique.add(candidate);
                nodes.add(candidate);
                if (unique.size() == count) {
                    break;
                }
            }
        }
        nextEntries = candidates;
    }
    return nodes.build();
}

Example 25 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class ClusterStatsResource method proxyClusterStats.

private void proxyClusterStats(HttpServletRequest servletRequest, AsyncResponse asyncResponse, String xForwardedProto, UriInfo uriInfo) {
    try {
        checkState(proxyHelper.isPresent());
        Iterator<InternalNode> resourceManagers = nodeManager.getResourceManagers().iterator();
        if (!resourceManagers.hasNext()) {
            asyncResponse.resume(Response.status(SERVICE_UNAVAILABLE).build());
            return;
        }
        InternalNode resourceManagerNode = resourceManagers.next();
        URI uri = uriInfo.getRequestUriBuilder().scheme(resourceManagerNode.getInternalUri().getScheme()).host(resourceManagerNode.getHostAndPort().toInetAddress().getHostName()).port(resourceManagerNode.getInternalUri().getPort()).build();
        proxyHelper.get().performRequest(servletRequest, asyncResponse, uri);
    } catch (Exception e) {
        asyncResponse.resume(e);
    }
}