Examples with InternalNode com.facebook.presto.metadata.InternalNode used on opensource projects

Example 61 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class SourcePartitionedScheduler method assignSplits.

private Set<RemoteTask> assignSplits(Multimap<InternalNode, Split> splitAssignment, Multimap<InternalNode, Lifespan> noMoreSplitsNotification) {
    ImmutableSet.Builder<RemoteTask> newTasks = ImmutableSet.builder();
    ImmutableSet<InternalNode> nodes = ImmutableSet.<InternalNode>builder().addAll(splitAssignment.keySet()).addAll(noMoreSplitsNotification.keySet()).build();
    for (InternalNode node : nodes) {
        ImmutableMultimap<PlanNodeId, Split> splits = ImmutableMultimap.<PlanNodeId, Split>builder().putAll(partitionedNode, splitAssignment.get(node)).build();
        ImmutableMultimap.Builder<PlanNodeId, Lifespan> noMoreSplits = ImmutableMultimap.builder();
        if (noMoreSplitsNotification.containsKey(node)) {
            noMoreSplits.putAll(partitionedNode, noMoreSplitsNotification.get(node));
        }
        newTasks.addAll(stage.scheduleSplits(node, splits, noMoreSplits.build()));
    }
    return newTasks.build();
}

Example 62 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class FixedLifespanScheduler method scheduleInitial.

public void scheduleInitial(SourceScheduler scheduler) {
    checkState(!initialScheduled);
    initialScheduled = true;
    for (Map.Entry<InternalNode, IntListIterator> entry : nodeToDriverGroupsMap.entrySet()) {
        IntListIterator driverGroupsIterator = entry.getValue();
        int driverGroupsScheduled = 0;
        while (driverGroupsIterator.hasNext()) {
            int driverGroupId = driverGroupsIterator.nextInt();
            scheduler.startLifespan(Lifespan.driverGroup(driverGroupId), partitionHandles.get(driverGroupId));
            driverGroupsScheduled++;
            if (concurrentLifespansPerTask.isPresent() && driverGroupsScheduled == concurrentLifespansPerTask.getAsInt()) {
                break;
            }
        }
    }
}

Example 63 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class ClusterMemoryManager method updateNodes.

private synchronized void updateNodes(MemoryPoolAssignmentsRequest assignments) {
    ImmutableSet.Builder<InternalNode> builder = ImmutableSet.builder();
    Set<InternalNode> aliveNodes = builder.addAll(nodeManager.getNodes(ACTIVE)).addAll(nodeManager.getNodes(SHUTTING_DOWN)).build();
    ImmutableSet<String> aliveNodeIds = aliveNodes.stream().map(InternalNode::getNodeIdentifier).collect(toImmutableSet());
    // Remove nodes that don't exist anymore
    // Make a copy to materialize the set difference
    Set<String> deadNodes = ImmutableSet.copyOf(difference(nodes.keySet(), aliveNodeIds));
    nodes.keySet().removeAll(deadNodes);
    // Add new nodes
    for (InternalNode node : aliveNodes) {
        if (!nodes.containsKey(node.getNodeIdentifier())) {
            nodes.put(node.getNodeIdentifier(), new RemoteNodeMemory(node, httpClient, memoryInfoCodec, assignmentsRequestCodec, locationFactory.createMemoryInfoLocation(node), isBinaryTransportEnabled));
        }
    }
    // in polling or updating (when moving queries to the reserved pool) its memory pools
    if (!isWorkScheduledOnCoordinator) {
        nodes.remove(nodeManager.getCurrentNode().getNodeIdentifier());
    }
    // Schedule refresh
    for (RemoteNodeMemory node : nodes.values()) {
        node.asyncRefresh(assignments);
    }
}

Example 64 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestSqlStageExecution method testFinalStageInfoInternal.

private void testFinalStageInfoInternal() throws Exception {
    NodeTaskMap nodeTaskMap = new NodeTaskMap(new FinalizerService());
    StageId stageId = new StageId(new QueryId("query"), 0);
    SqlStageExecution stage = createSqlStageExecution(new StageExecutionId(stageId, 0), createExchangePlanFragment(), new MockRemoteTaskFactory(executor, scheduledExecutor), TEST_SESSION, true, nodeTaskMap, executor, new NoOpFailureDetector(), new SplitSchedulerStats(), new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()));
    stage.setOutputBuffers(createInitialEmptyOutputBuffers(ARBITRARY));
    // add listener that fetches stage info when the final status is available
    SettableFuture<StageExecutionInfo> finalStageInfo = SettableFuture.create();
    stage.addFinalStageInfoListener(finalStageInfo::set);
    // in a background thread add a ton of tasks
    CountDownLatch latch = new CountDownLatch(1000);
    Future<?> addTasksTask = executor.submit(() -> {
        try {
            for (int i = 0; i < 1_000_000; i++) {
                if (Thread.interrupted()) {
                    return;
                }
                InternalNode node = new InternalNode("source" + i, URI.create("http://10.0.0." + (i / 10_000) + ":" + (i % 10_000)), NodeVersion.UNKNOWN, false);
                stage.scheduleTask(node, i);
                latch.countDown();
            }
        } finally {
            while (latch.getCount() > 0) {
                latch.countDown();
            }
        }
    });
    // wait for some tasks to be created, and then abort the query
    latch.await(1, MINUTES);
    assertFalse(stage.getStageExecutionInfo().getTasks().isEmpty());
    stage.abort();
    // once the final stage info is available, verify that it is complete
    StageExecutionInfo stageInfo = finalStageInfo.get(1, MINUTES);
    assertFalse(stageInfo.getTasks().isEmpty());
    assertTrue(stageInfo.isFinal());
    assertSame(stage.getStageExecutionInfo(), stageInfo);
    // cancel the background thread adding tasks
    addTasksTask.cancel(true);
}

Example 65 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestConsistentHashingNodeProvider method testDistribution.

@Test
public void testDistribution() {
    List<InternalNode> nodes = IntStream.range(0, 10).mapToObj(i -> new InternalNode(format("other%d", i), URI.create(format("http://127.0.0.%d:100", i)), NodeVersion.UNKNOWN, false)).collect(toImmutableList());
    ConsistentHashingNodeProvider nodeProvider = ConsistentHashingNodeProvider.create(nodes, 100);
    Random random = new Random();
    Map<HostAddress, Integer> result = new HashMap<>();
    for (int i = 0; i < 1_000_000; i++) {
        List<HostAddress> candidates = nodeProvider.get(format("split%d", random.nextInt()), 2);
        assertNotEquals(candidates.get(1), candidates.get(0));
        HostAddress hostAddress = candidates.get(0);
        int count = result.getOrDefault(hostAddress, 0);
        result.put(hostAddress, count + 1);
    }
    assertTrue(result.values().stream().allMatch(count -> count >= 80000 && count <= 120000));
}