Examples with InternalNode com.facebook.presto.metadata.InternalNode used on opensource projects

Example 66 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class LowMemoryKillerTestingUtils method toNodeMemoryInfoList.

static List<MemoryInfo> toNodeMemoryInfoList(long maxReservedPoolBytes, long maxGeneralPoolBytes, String reservedQuery, Map<String, Map<String, Long>> queries) {
    Map<InternalNode, NodeReservation> nodeReservations = new HashMap<>();
    for (Map.Entry<String, Map<String, Long>> entry : queries.entrySet()) {
        QueryId queryId = new QueryId(entry.getKey());
        Map<String, Long> reservationByNode = entry.getValue();
        for (Map.Entry<String, Long> nodeEntry : reservationByNode.entrySet()) {
            InternalNode node = new InternalNode(nodeEntry.getKey(), URI.create("http://localhost"), new NodeVersion("version"), false);
            long bytes = nodeEntry.getValue();
            if (bytes == 0) {
                continue;
            }
            if (reservedQuery.equals(entry.getKey())) {
                nodeReservations.computeIfAbsent(node, ignored -> new NodeReservation()).getReserved().add(queryId, bytes);
            } else {
                nodeReservations.computeIfAbsent(node, ignored -> new NodeReservation()).getGeneral().add(queryId, bytes);
            }
        }
    }
    ImmutableList.Builder<MemoryInfo> result = ImmutableList.builder();
    for (Map.Entry<InternalNode, NodeReservation> entry : nodeReservations.entrySet()) {
        NodeReservation nodeReservation = entry.getValue();
        ImmutableMap.Builder<MemoryPoolId, MemoryPoolInfo> pools = ImmutableMap.builder();
        if (nodeReservation.getGeneral().getTotalReservedBytes() > 0) {
            pools.put(GENERAL_POOL, new MemoryPoolInfo(maxGeneralPoolBytes, nodeReservation.getGeneral().getTotalReservedBytes(), 0, nodeReservation.getGeneral().getReservationByQuery(), ImmutableMap.of(), ImmutableMap.of()));
        }
        if (nodeReservation.getReserved().getTotalReservedBytes() > 0) {
            pools.put(RESERVED_POOL, new MemoryPoolInfo(maxReservedPoolBytes, nodeReservation.getReserved().getTotalReservedBytes(), 0, nodeReservation.getReserved().getReservationByQuery(), ImmutableMap.of(), ImmutableMap.of()));
        }
        result.add(new MemoryInfo(new DataSize(maxReservedPoolBytes + maxGeneralPoolBytes, BYTE), pools.build()));
    }
    return result.build();
}

Example 67 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class QueryStateInfoResource method proxyQueryStateInfo.

// TODO the pattern of this function is similar with ClusterStatsResource and QueryResource, we can move it to a common place and re-use.
private void proxyQueryStateInfo(HttpServletRequest servletRequest, AsyncResponse asyncResponse, String xForwardedProto, UriInfo uriInfo) {
    try {
        checkState(proxyHelper.isPresent());
        Iterator<InternalNode> resourceManagers = internalNodeManager.getResourceManagers().iterator();
        if (!resourceManagers.hasNext()) {
            asyncResponse.resume(Response.status(SERVICE_UNAVAILABLE).build());
            return;
        }
        InternalNode resourceManagerNode = resourceManagers.next();
        URI uri = uriInfo.getRequestUriBuilder().scheme(resourceManagerNode.getInternalUri().getScheme()).host(resourceManagerNode.getHostAndPort().toInetAddress().getHostName()).port(resourceManagerNode.getInternalUri().getPort()).build();
        proxyHelper.get().performRequest(servletRequest, asyncResponse, uri);
    } catch (Exception e) {
        asyncResponse.resume(e);
    }
}

Example 68 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class ResourceGroupStateInfoResource method proxyResourceGroupInfoResponse.

// TODO move this to a common place and reuse in all resource
private void proxyResourceGroupInfoResponse(HttpServletRequest servletRequest, AsyncResponse asyncResponse, String xForwardedProto, UriInfo uriInfo) {
    try {
        checkState(proxyHelper.isPresent());
        Iterator<InternalNode> resourceManagers = internalNodeManager.getResourceManagers().iterator();
        if (!resourceManagers.hasNext()) {
            asyncResponse.resume(Response.status(SERVICE_UNAVAILABLE).build());
            return;
        }
        InternalNode resourceManagerNode = resourceManagers.next();
        URI uri = uriInfo.getRequestUriBuilder().scheme(resourceManagerNode.getInternalUri().getScheme()).host(resourceManagerNode.getHostAndPort().toInetAddress().getHostName()).port(resourceManagerNode.getInternalUri().getPort()).build();
        proxyHelper.get().performRequest(servletRequest, asyncResponse, uri);
    } catch (Exception e) {
        asyncResponse.resume(e);
    }
}

Example 69 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestResourceManagerClusterStatusSender method setup.

@BeforeTest
public void setup() {
    resourceManagerClient = new TestingResourceManagerClient();
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(CONNECTOR_ID, new InternalNode("identifier", URI.create("http://localhost:80/identifier"), OptionalInt.of(1), "1", false, true));
    sender = new ResourceManagerClusterStatusSender((addressSelectionContext, headers) -> resourceManagerClient, nodeManager, () -> NODE_STATUS, newSingleThreadScheduledExecutor(), new ResourceManagerConfig().setNodeHeartbeatInterval(new Duration(HEARTBEAT_INTERVAL, MILLISECONDS)).setQueryHeartbeatInterval(new Duration(HEARTBEAT_INTERVAL, MILLISECONDS)));
}

Example 70 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class NodePartitioningManager method getNodePartitioningMap.

public NodePartitionMap getNodePartitioningMap(Session session, PartitioningHandle partitioningHandle) {
    requireNonNull(session, "session is null");
    requireNonNull(partitioningHandle, "partitioningHandle is null");
    if (partitioningHandle.getConnectorHandle() instanceof SystemPartitioningHandle) {
        return ((SystemPartitioningHandle) partitioningHandle.getConnectorHandle()).getNodePartitionMap(session, nodeScheduler);
    }
    ConnectorId connectorId = partitioningHandle.getConnectorId().orElseThrow(() -> new IllegalArgumentException("No connector ID for partitioning handle: " + partitioningHandle));
    ConnectorBucketNodeMap connectorBucketNodeMap = getConnectorBucketNodeMap(session, partitioningHandle);
    // safety check for crazy partitioning
    checkArgument(connectorBucketNodeMap.getBucketCount() < 1_000_000, "Too many buckets in partitioning: %s", connectorBucketNodeMap.getBucketCount());
    List<InternalNode> bucketToNode;
    NodeSelectionStrategy nodeSelectionStrategy = connectorBucketNodeMap.getNodeSelectionStrategy();
    boolean cacheable;
    switch(nodeSelectionStrategy) {
        case HARD_AFFINITY:
            bucketToNode = getFixedMapping(connectorBucketNodeMap);
            cacheable = false;
            break;
        case SOFT_AFFINITY:
            bucketToNode = getFixedMapping(connectorBucketNodeMap);
            cacheable = true;
            break;
        case NO_PREFERENCE:
            bucketToNode = createArbitraryBucketToNode(nodeScheduler.createNodeSelector(session, connectorId).selectRandomNodes(getMaxTasksPerStage(session)), connectorBucketNodeMap.getBucketCount());
            cacheable = false;
            break;
        default:
            throw new PrestoException(NODE_SELECTION_NOT_SUPPORTED, format("Unsupported node selection strategy %s", nodeSelectionStrategy));
    }
    int[] bucketToPartition = new int[connectorBucketNodeMap.getBucketCount()];
    BiMap<InternalNode, Integer> nodeToPartition = HashBiMap.create();
    int nextPartitionId = 0;
    for (int bucket = 0; bucket < bucketToNode.size(); bucket++) {
        InternalNode node = bucketToNode.get(bucket);
        Integer partitionId = nodeToPartition.get(node);
        if (partitionId == null) {
            partitionId = nextPartitionId++;
            nodeToPartition.put(node, partitionId);
        }
        bucketToPartition[bucket] = partitionId;
    }
    List<InternalNode> partitionToNode = IntStream.range(0, nodeToPartition.size()).mapToObj(partitionId -> nodeToPartition.inverse().get(partitionId)).collect(toImmutableList());
    return new NodePartitionMap(partitionToNode, bucketToPartition, getSplitToBucket(session, partitioningHandle), cacheable);
}