Examples with InternalNode com.facebook.presto.metadata.InternalNode used on opensource projects

Example 11 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class QueryResource method proxyResponse.

private void proxyResponse(HttpServletRequest servletRequest, AsyncResponse asyncResponse, String xForwardedProto, UriInfo uriInfo) {
    try {
        checkState(proxyHelper.isPresent());
        Iterator<InternalNode> resourceManagers = internalNodeManager.getResourceManagers().iterator();
        if (!resourceManagers.hasNext()) {
            asyncResponse.resume(Response.status(SERVICE_UNAVAILABLE).build());
            return;
        }
        InternalNode resourceManagerNode = resourceManagers.next();
        URI uri = uriInfo.getRequestUriBuilder().scheme(resourceManagerNode.getInternalUri().getScheme()).host(resourceManagerNode.getHostAndPort().toInetAddress().getHostName()).port(resourceManagerNode.getInternalUri().getPort()).build();
        proxyHelper.get().performRequest(servletRequest, asyncResponse, uri);
    } catch (Exception e) {
        asyncResponse.resume(e);
    }
}

Example 12 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class WorkerResource method proxyJsonResponse.

private Response proxyJsonResponse(String nodeId, String workerPath) {
    Set<InternalNode> nodes = nodeManager.getNodes(NodeState.ACTIVE);
    InternalNode node = nodes.stream().filter(n -> n.getNodeIdentifier().equals(nodeId)).findFirst().orElseThrow(() -> new WebApplicationException(NOT_FOUND));
    Request request = prepareGet().setUri(uriBuilderFrom(node.getInternalUri()).appendPath(workerPath).build()).build();
    InputStream responseStream = httpClient.execute(request, new StreamingJsonResponseHandler());
    return Response.ok(responseStream, APPLICATION_JSON_TYPE).build();
}

Example 13 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestNodeScheduler method setUp.

@BeforeMethod
public void setUp() {
    session = TestingSession.testSessionBuilder().build();
    finalizerService = new FinalizerService();
    nodeTaskMap = new NodeTaskMap(finalizerService);
    nodeManager = new InMemoryNodeManager();
    ImmutableList.Builder<InternalNode> nodeBuilder = ImmutableList.builder();
    nodeBuilder.add(new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false));
    nodeBuilder.add(new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false));
    nodeBuilder.add(new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false));
    List<InternalNode> nodes = nodeBuilder.build();
    nodeManager.addNode(CONNECTOR_ID, nodes);
    nodeSchedulerConfig = new NodeSchedulerConfig().setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
    nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
    // contents of taskMap indicate the node-task map for the current stage
    taskMap = new HashMap<>();
    nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID);
    remoteTaskExecutor = newCachedThreadPool(daemonThreadsNamed("remoteTaskExecutor-%s"));
    remoteTaskScheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("remoteTaskScheduledExecutor-%s"));
    finalizerService.start();
}

Example 14 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestNodeScheduler method testBasicAssignment.

@Test
public void testBasicAssignment() {
    // One split for each node
    Set<Split> splits = new HashSet<>();
    for (int i = 0; i < 3; i++) {
        splits.add(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote()));
    }
    Multimap<InternalNode, Split> assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments();
    assertEquals(assignments.entries().size(), 3);
    for (InternalNode node : nodeManager.getActiveConnectorNodes(CONNECTOR_ID)) {
        assertTrue(assignments.keySet().contains(node));
    }
}

Example 15 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestNodeScheduler method testAffinityAssignmentWithModularHashing.

@Test
public void testAffinityAssignmentWithModularHashing() {
    NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
    TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
    NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
    NodeSelector nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 3);
    Set<Split> splits = new HashSet<>();
    // Adding one more split (1 % 3 = 1), 1 splits will be distributed to 1 nodes
    splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestAffinitySplitRemote(1)));
    SplitPlacementResult splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of());
    Set<InternalNode> internalNodes = splitPlacementResult.getAssignments().keySet();
    assertEquals(internalNodes.size(), 1);
    // Adding one more split (2 % 3 = 2), 2 splits will be distributed to 2 nodes
    splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestAffinitySplitRemote(2)));
    splitPlacementResult = nodeSelector.computeAssignments(splits, getRemoteTableScanTask(splitPlacementResult));
    Set<InternalNode> internalNodesSecondCall = splitPlacementResult.getAssignments().keySet();
    assertEquals(internalNodesSecondCall.size(), 2);
    // adding one more split(4 % 3 = 1) that will fall into the same slots, 3 splits will be distributed to 2 nodes still
    splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestAffinitySplitRemote(4)));
    splitPlacementResult = nodeSelector.computeAssignments(splits, getRemoteTableScanTask(splitPlacementResult));
    assertEquals(splitPlacementResult.getAssignments().keySet().size(), 2);
    // adding one more split(3 % 3 = 0) that will fall into different slots, 3 splits will be distributed to 3 nodes
    splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestAffinitySplitRemote(3)));
    splitPlacementResult = nodeSelector.computeAssignments(splits, getRemoteTableScanTask(splitPlacementResult));
    assertEquals(splitPlacementResult.getAssignments().keySet().size(), 3);
}