Examples with ClusterMemoryPoolInfo com.facebook.presto.spi.memory.ClusterMemoryPoolInfo used on opensource projects

Example 11 with ClusterMemoryPoolInfo

use of com.facebook.presto.spi.memory.ClusterMemoryPoolInfo in project urban-eureka by errir503.

the class ClusterMemoryManager method updateAssignments.

// TODO once the reserved pool is removed we can remove this method. We can also update
// RemoteNodeMemory as we don't need to POST anything.
private synchronized MemoryPoolAssignmentsRequest updateAssignments(Iterable<QueryExecution> queries) {
    ClusterMemoryPoolInfo reservedPool = getClusterInfo(RESERVED_POOL);
    ClusterMemoryPoolInfo generalPool = getClusterInfo(GENERAL_POOL);
    verify(generalPool != null, "generalPool is null");
    verify(reservedPool != null, "reservedPool is null");
    long version = memoryPoolAssignmentsVersion.incrementAndGet();
    // and is more of a safety check than a guarantee
    if (allAssignmentsHavePropagated(queries)) {
        if (reservedPool.getAssignedQueries() == 0 && generalPool.getBlockedNodes() > 0) {
            QueryExecution biggestQuery = findLargestMemoryQuery(generalPool, queries);
            if (biggestQuery != null) {
                log.info("Moving query %s to the reserved pool", biggestQuery.getQueryId());
                biggestQuery.setMemoryPool(new VersionedMemoryPoolId(RESERVED_POOL, version));
            }
        }
    }
    ImmutableList.Builder<MemoryPoolAssignment> assignments = ImmutableList.builder();
    for (QueryExecution queryExecution : queries) {
        assignments.add(new MemoryPoolAssignment(queryExecution.getQueryId(), queryExecution.getMemoryPool().getId()));
    }
    return new MemoryPoolAssignmentsRequest(coordinatorId, version, assignments.build());
}

Example 12 with ClusterMemoryPoolInfo

use of com.facebook.presto.spi.memory.ClusterMemoryPoolInfo in project urban-eureka by errir503.

the class ClusterMemoryManager method findLargestMemoryQuery.

private QueryExecution findLargestMemoryQuery(ClusterMemoryPoolInfo generalPool, Iterable<QueryExecution> queries) {
    QueryExecution biggestQuery = null;
    long maxMemory = -1;
    Optional<QueryId> largestMemoryQuery = generalPool.getLargestMemoryQuery();
    // If present, this means the resource manager is determining the largest query, so do not make this determination locally
    if (memoryManagerService.isPresent()) {
        return largestMemoryQuery.flatMap(largestMemoryQueryId -> Streams.stream(queries).filter(query -> query.getQueryId().equals(largestMemoryQueryId)).findFirst()).orElse(null);
    }
    for (QueryExecution queryExecution : queries) {
        if (resourceOvercommit(queryExecution.getSession())) {
            // since their memory usage is unbounded.
            continue;
        }
        long bytesUsed = getQueryMemoryReservation(queryExecution);
        if (bytesUsed > maxMemory) {
            biggestQuery = queryExecution;
            maxMemory = bytesUsed;
        }
    }
    return biggestQuery;
}