Examples with InternalNode io.prestosql.metadata.InternalNode used on opensource projects

Example 26 with InternalNode

use of io.prestosql.metadata.InternalNode in project hetu-core by openlookeng.

the class SystemSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableHandle tableHandle, SplitSchedulingStrategy splitSchedulingStrategy) {
    SystemTableHandle table = (SystemTableHandle) tableHandle;
    TupleDomain<ColumnHandle> constraint = table.getConstraint();
    SystemTable systemTable = tables.getSystemTable(session, table.getSchemaTableName()).orElseThrow(() -> new TableNotFoundException(table.getSchemaTableName()));
    Distribution tableDistributionMode = systemTable.getDistribution();
    if (tableDistributionMode == SINGLE_COORDINATOR) {
        HostAddress address = nodeManager.getCurrentNode().getHostAndPort();
        ConnectorSplit split = new SystemSplit(address, constraint);
        return new FixedSplitSource(ImmutableList.of(split));
    }
    ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder();
    ImmutableSet.Builder<InternalNode> nodes = ImmutableSet.builder();
    if (tableDistributionMode == ALL_COORDINATORS) {
        nodes.addAll(nodeManager.getCoordinators());
    } else if (tableDistributionMode == ALL_NODES) {
        nodes.addAll(nodeManager.getNodes(ACTIVE));
    }
    Set<InternalNode> nodeSet = nodes.build();
    for (InternalNode node : nodeSet) {
        splits.add(new SystemSplit(node.getHostAndPort(), constraint));
    }
    return new FixedSplitSource(splits.build());
}

Example 27 with InternalNode

use of io.prestosql.metadata.InternalNode in project hetu-core by openlookeng.

the class ClusterMemoryManager method updateNodes.

private synchronized void updateNodes(MemoryPoolAssignmentsRequest assignments) {
    ImmutableSet.Builder<InternalNode> builder = ImmutableSet.builder();
    Set<InternalNode> aliveNodes = builder.addAll(nodeManager.getNodes(ACTIVE)).addAll(nodeManager.getNodes(ISOLATING)).addAll(nodeManager.getNodes(ISOLATED)).addAll(nodeManager.getNodes(SHUTTING_DOWN)).build();
    ImmutableSet<String> aliveNodeIds = aliveNodes.stream().map(InternalNode::getNodeIdentifier).collect(toImmutableSet());
    // Remove nodes that don't exist anymore
    // Make a copy to materialize the set difference
    Set<String> deadNodes = ImmutableSet.copyOf(difference(nodes.keySet(), aliveNodeIds));
    nodes.keySet().removeAll(deadNodes);
    // Add new nodes
    for (InternalNode node : aliveNodes) {
        if (!nodes.containsKey(node.getNodeIdentifier()) && shouldIncludeNode(node)) {
            nodes.put(node.getInternalUri().toString(), new RemoteNodeMemory(node, httpClient, memoryInfoCodec, assignmentsRequestCodec, locationFactory.createMemoryInfoLocation(node), isBinaryEncoding));
            allNodes.put(node.getInternalUri().toString(), new RemoteNodeMemory(node, httpClient, memoryInfoCodec, assignmentsRequestCodec, locationFactory.createMemoryInfoLocation(node), isBinaryEncoding));
        }
    }
    // Schedule refresh
    for (RemoteNodeMemory node : nodes.values()) {
        node.asyncRefresh(assignments);
    }
    // Schedule All refresh
    for (RemoteNodeMemory node : allNodes.values()) {
        node.asyncRefresh(assignments);
    }
}

Example 28 with InternalNode

use of io.prestosql.metadata.InternalNode in project hetu-core by openlookeng.

the class ClusterMemoryManager method getWorkerMemoryInfo.

public synchronized Map<String, Optional<MemoryInfo>> getWorkerMemoryInfo() {
    Map<String, Optional<MemoryInfo>> memoryInfo = new HashMap<>();
    for (Entry<String, RemoteNodeMemory> entry : nodes.entrySet()) {
        // workerId is of the form "node_identifier [node_host] role"
        InternalNode node = entry.getValue().getNode();
        String role = node.isCoordinator() ? (node.isWorker() ? "Coordinator & Worker" : "Coordinator") : "Worker";
        String workerId = entry.getKey() + " [" + entry.getValue().getNode().getHost() + "] " + role;
        memoryInfo.put(workerId, entry.getValue().getInfo());
    }
    return memoryInfo;
}

Example 29 with InternalNode

use of io.prestosql.metadata.InternalNode in project hetu-core by openlookeng.

the class BenchmarkNodeScheduler method benchmark.

@Benchmark
@OperationsPerInvocation(SPLITS)
public Object benchmark(BenchmarkData data) {
    List<RemoteTask> remoteTasks = ImmutableList.copyOf(data.getTaskMap().values());
    Iterator<MockRemoteTaskFactory.MockRemoteTask> finishingTask = Iterators.cycle(data.getTaskMap().values());
    Iterator<Split> splits = data.getSplits().iterator();
    Set<Split> batch = new HashSet<>();
    while (splits.hasNext() || !batch.isEmpty()) {
        Multimap<InternalNode, Split> assignments = data.getNodeSelector().computeAssignments(batch, remoteTasks, Optional.empty()).getAssignments();
        for (InternalNode node : assignments.keySet()) {
            MockRemoteTaskFactory.MockRemoteTask remoteTask = data.getTaskMap().get(node);
            remoteTask.addSplits(ImmutableMultimap.<PlanNodeId, Split>builder().putAll(new PlanNodeId("sourceId"), assignments.get(node)).build());
            remoteTask.startSplits(MAX_SPLITS_PER_NODE);
        }
        if (assignments.size() == batch.size()) {
            batch.clear();
        } else {
            batch.removeAll(assignments.values());
        }
        while (batch.size() < SPLIT_BATCH_SIZE && splits.hasNext()) {
            batch.add(splits.next());
        }
        finishingTask.next().finishSplits((int) Math.ceil(MAX_SPLITS_PER_NODE / 50.0));
    }
    return remoteTasks;
}

Example 30 with InternalNode

use of io.prestosql.metadata.InternalNode in project hetu-core by openlookeng.

the class DynamicFilterService method registerTasksHelper.

private void registerTasksHelper(PlanNode node, Symbol buildSymbol, Map<String, Symbol> dynamicFiltersMap, Set<TaskId> taskIds, Set<InternalNode> workers, StageStateMachine stateMachine) {
    final StateStore stateStore = stateStoreProvider.getStateStore();
    String queryId = stateMachine.getSession().getQueryId().toString();
    for (Map.Entry<String, Symbol> entry : dynamicFiltersMap.entrySet()) {
        Symbol buildSymbolToCheck = buildSymbol != null ? buildSymbol : node.getOutputSymbols().contains(entry.getValue()) ? entry.getValue() : null;
        if (buildSymbolToCheck != null && entry.getValue().getName().equals(buildSymbol.getName())) {
            String filterId = entry.getKey();
            stateStore.createStateCollection(createKey(DynamicFilterUtils.TASKSPREFIX, filterId, queryId), SET);
            stateStore.createStateCollection(createKey(DynamicFilterUtils.PARTIALPREFIX, filterId, queryId), SET);
            dynamicFilters.putIfAbsent(queryId, new ConcurrentHashMap<>());
            Map<String, DynamicFilterRegistryInfo> filters = dynamicFilters.get(queryId);
            if (node instanceof JoinNode) {
                filters.put(filterId, extractDynamicFilterRegistryInfo((JoinNode) node, stateMachine.getSession(), filterId));
            } else if (node instanceof SemiJoinNode) {
                filters.put(filterId, extractDynamicFilterRegistryInfo((SemiJoinNode) node, stateMachine.getSession()));
            }
            dynamicFiltersToTask.putIfAbsent(filterId + "-" + queryId, new CopyOnWriteArraySet<>());
            CopyOnWriteArraySet<TaskId> taskSet = dynamicFiltersToTask.get(filterId + "-" + queryId);
            taskSet.addAll(taskIds);
            log.debug("registerTasks source " + filterId + " filters:" + filters + ", workers: " + workers.stream().map(x -> x.getNodeIdentifier()).collect(Collectors.joining(",")) + ", taskIds: " + taskIds.stream().map(TaskId::toString).collect(Collectors.joining(",")));
        }
    }
}