Examples with MemoryPoolId com.facebook.presto.spi.memory.MemoryPoolId used on opensource projects

Example 6 with MemoryPoolId

use of com.facebook.presto.spi.memory.MemoryPoolId in project presto by prestodb.

the class TestMemoryPools method setUp.

private void setUp(Supplier<List<Driver>> driversSupplier) {
    checkState(localQueryRunner == null, "Already set up");
    Session session = testSessionBuilder().setCatalog("tpch").setSchema("tiny").setSystemProperty("task_default_concurrency", "1").build();
    localQueryRunner = queryRunnerWithInitialTransaction(session);
    // add tpch
    localQueryRunner.createCatalog("tpch", new TpchConnectorFactory(1), ImmutableMap.of());
    userPool = new MemoryPool(new MemoryPoolId("test"), TEN_MEGABYTES);
    fakeQueryId = new QueryId("fake");
    SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(new DataSize(1, GIGABYTE));
    QueryContext queryContext = new QueryContext(new QueryId("query"), TEN_MEGABYTES, new DataSize(20, MEGABYTE), TEN_MEGABYTES, new DataSize(1, GIGABYTE), userPool, new TestingGcMonitor(), localQueryRunner.getExecutor(), localQueryRunner.getScheduler(), TEN_MEGABYTES, spillSpaceTracker, listJsonCodec(TaskMemoryReservationSummary.class));
    taskContext = createTaskContext(queryContext, localQueryRunner.getExecutor(), session);
    drivers = driversSupplier.get();
}

Example 7 with MemoryPoolId

use of com.facebook.presto.spi.memory.MemoryPoolId in project presto by prestodb.

the class TestMemoryPools method testTaggedAllocations.

@Test
public void testTaggedAllocations() {
    QueryId testQuery = new QueryId("test_query");
    MemoryPool testPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(1000, BYTE));
    testPool.reserve(testQuery, "test_tag", 10);
    Map<String, Long> allocations = testPool.getTaggedMemoryAllocations(testQuery);
    assertEquals(allocations, ImmutableMap.of("test_tag", 10L));
    // free 5 bytes for test_tag
    testPool.free(testQuery, "test_tag", 5);
    allocations = testPool.getTaggedMemoryAllocations(testQuery);
    assertEquals(allocations, ImmutableMap.of("test_tag", 5L));
    testPool.reserve(testQuery, "test_tag2", 20);
    allocations = testPool.getTaggedMemoryAllocations(testQuery);
    assertEquals(allocations, ImmutableMap.of("test_tag", 5L, "test_tag2", 20L));
    // free the remaining 5 bytes for test_tag
    testPool.free(testQuery, "test_tag", 5);
    allocations = testPool.getTaggedMemoryAllocations(testQuery);
    assertEquals(allocations, ImmutableMap.of("test_tag2", 20L));
    // free all for test_tag2
    testPool.free(testQuery, "test_tag2", 20);
    assertEquals(testPool.getTaggedMemoryAllocations().size(), 0);
}

Example 8 with MemoryPoolId

use of com.facebook.presto.spi.memory.MemoryPoolId in project presto by prestodb.

the class TestMemoryTracking method setUpTest.

@BeforeMethod
public void setUpTest() {
    memoryPool = new MemoryPool(new MemoryPoolId("test"), memoryPoolSize);
    queryContext = new QueryContext(new QueryId("test_query"), queryMaxMemory, queryMaxTotalMemory, queryMaxMemory, queryMaxRevocableMemory, memoryPool, new TestingGcMonitor(), notificationExecutor, yieldExecutor, queryMaxSpillSize, spillSpaceTracker, listJsonCodec(TaskMemoryReservationSummary.class));
    taskContext = queryContext.addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0, 0), notificationExecutor), testSessionBuilder().build(), Optional.of(PLAN_FRAGMENT.getRoot()), true, true, true, true, false);
    pipelineContext = taskContext.addPipelineContext(0, true, true, false);
    driverContext = pipelineContext.addDriverContext();
    operatorContext = driverContext.addOperatorContext(1, new PlanNodeId("a"), "test-operator");
}

Example 9 with MemoryPoolId

use of com.facebook.presto.spi.memory.MemoryPoolId in project presto by prestodb.

the class MemoryLocalQueryRunner method execute.

public List<Page> execute(@Language("SQL") String query) {
    MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(2, GIGABYTE));
    SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(new DataSize(1, GIGABYTE));
    QueryContext queryContext = new QueryContext(new QueryId("test"), new DataSize(1, GIGABYTE), new DataSize(2, GIGABYTE), new DataSize(1, GIGABYTE), new DataSize(2, GIGABYTE), memoryPool, new TestingGcMonitor(), localQueryRunner.getExecutor(), localQueryRunner.getScheduler(), new DataSize(4, GIGABYTE), spillSpaceTracker, listJsonCodec(TaskMemoryReservationSummary.class));
    TaskContext taskContext = queryContext.addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0, 0), localQueryRunner.getExecutor()), localQueryRunner.getDefaultSession(), Optional.empty(), false, false, false, false, false);
    // Use NullOutputFactory to avoid coping out results to avoid affecting benchmark results
    ImmutableList.Builder<Page> output = ImmutableList.builder();
    List<Driver> drivers = localQueryRunner.createDrivers(query, new PageConsumerOperator.PageConsumerOutputFactory(types -> output::add), taskContext);
    boolean done = false;
    while (!done) {
        boolean processed = false;
        for (Driver driver : drivers) {
            if (!driver.isFinished()) {
                driver.process();
                processed = true;
            }
        }
        done = !processed;
    }
    return output.build();
}

Example 10 with MemoryPoolId

use of com.facebook.presto.spi.memory.MemoryPoolId in project presto by prestodb.

the class ClusterMemoryManager method process.

public synchronized void process(Iterable<QueryExecution> runningQueries, Supplier<List<BasicQueryInfo>> allQueryInfoSupplier) {
    if (!enabled) {
        return;
    }
    // TODO revocable memory reservations can also leak and may need to be detected in the future
    // We are only concerned about the leaks in general pool.
    memoryLeakDetector.checkForMemoryLeaks(allQueryInfoSupplier, pools.get(GENERAL_POOL).getQueryMemoryReservations());
    boolean outOfMemory = isClusterOutOfMemory();
    if (!outOfMemory) {
        lastTimeNotOutOfMemory = System.nanoTime();
    }
    boolean queryKilled = false;
    long totalUserMemoryBytes = 0L;
    long totalMemoryBytes = 0L;
    for (QueryExecution query : runningQueries) {
        boolean resourceOvercommit = resourceOvercommit(query.getSession());
        long userMemoryReservation = query.getUserMemoryReservation().toBytes();
        long totalMemoryReservation = query.getTotalMemoryReservation().toBytes();
        if (resourceOvercommit && outOfMemory) {
            // If a query has requested resource overcommit, only kill it if the cluster has run out of memory
            DataSize memory = succinctBytes(getQueryMemoryReservation(query));
            query.fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, format("The cluster is out of memory and %s=true, so this query was killed. It was using %s of memory", RESOURCE_OVERCOMMIT, memory)));
            queryKilled = true;
        }
        if (!resourceOvercommit) {
            long userMemoryLimit = min(maxQueryMemory.toBytes(), getQueryMaxMemory(query.getSession()).toBytes());
            if (userMemoryReservation > userMemoryLimit) {
                query.fail(exceededGlobalUserLimit(succinctBytes(userMemoryLimit)));
                queryKilled = true;
            }
            QueryLimit<DataSize> queryTotalMemoryLimit = getMinimum(createDataSizeLimit(maxQueryTotalMemory, SYSTEM), query.getResourceGroupQueryLimits().flatMap(ResourceGroupQueryLimits::getTotalMemoryLimit).map(rgLimit -> createDataSizeLimit(rgLimit, RESOURCE_GROUP)).orElse(null), createDataSizeLimit(getQueryMaxTotalMemory(query.getSession()), QUERY));
            if (totalMemoryReservation > queryTotalMemoryLimit.getLimit().toBytes()) {
                query.fail(exceededGlobalTotalLimit(queryTotalMemoryLimit.getLimit(), queryTotalMemoryLimit.getLimitSource().name()));
                queryKilled = true;
            }
        }
        totalUserMemoryBytes += userMemoryReservation;
        totalMemoryBytes += totalMemoryReservation;
    }
    clusterUserMemoryReservation.set(totalUserMemoryBytes);
    clusterTotalMemoryReservation.set(totalMemoryBytes);
    boolean killOnOomDelayPassed = nanosSince(lastTimeNotOutOfMemory).compareTo(killOnOutOfMemoryDelay) > 0;
    boolean lastKilledQueryGone = isLastKilledQueryGone();
    boolean shouldCallOomKiller = !(lowMemoryKiller instanceof NoneLowMemoryKiller) && outOfMemory && !queryKilled && killOnOomDelayPassed && lastKilledQueryGone;
    if (shouldCallOomKiller) {
        callOomKiller(runningQueries);
    } else {
        // if the cluster is out of memory and we didn't trigger the oom killer we log the state to make debugging easier
        if (outOfMemory) {
            log.debug("The cluster is out of memory and the OOM killer is not called (query killed: %s, kill on OOM delay passed: %s, last killed query gone: %s).", queryKilled, killOnOomDelayPassed, lastKilledQueryGone);
        }
    }
    Map<MemoryPoolId, Integer> countByPool = new HashMap<>();
    for (QueryExecution query : runningQueries) {
        MemoryPoolId id = query.getMemoryPool().getId();
        countByPool.put(id, countByPool.getOrDefault(id, 0) + 1);
    }
    updatePools(countByPool);
    MemoryPoolAssignmentsRequest assignmentsRequest;
    if (pools.containsKey(RESERVED_POOL)) {
        assignmentsRequest = updateAssignments(runningQueries);
    } else {
        // If reserved pool is not enabled, we don't create a MemoryPoolAssignmentsRequest that puts all the queries
        // in the general pool (as they already are). In this case we create an effectively NOOP MemoryPoolAssignmentsRequest.
        // Once the reserved pool is removed we should get rid of the logic of putting queries into reserved pool including
        // this piece of code.
        assignmentsRequest = new MemoryPoolAssignmentsRequest(coordinatorId, Long.MIN_VALUE, ImmutableList.of());
    }
    updateNodes(assignmentsRequest);
}