Examples with MemoryPoolId com.facebook.presto.spi.memory.MemoryPoolId used on opensource projects

Example 11 with MemoryPoolId

use of com.facebook.presto.spi.memory.MemoryPoolId in project presto by prestodb.

the class ClusterMemoryManager method createClusterMemoryPools.

private Map<MemoryPoolId, ClusterMemoryPool> createClusterMemoryPools(boolean reservedPoolEnabled) {
    Set<MemoryPoolId> memoryPools = new HashSet<>();
    memoryPools.add(GENERAL_POOL);
    if (reservedPoolEnabled) {
        memoryPools.add(RESERVED_POOL);
    }
    ImmutableMap.Builder<MemoryPoolId, ClusterMemoryPool> builder = ImmutableMap.builder();
    for (MemoryPoolId poolId : memoryPools) {
        ClusterMemoryPool pool = new ClusterMemoryPool(poolId);
        builder.put(poolId, pool);
        try {
            exporter.export(generatedNameOf(ClusterMemoryPool.class, poolId.toString()), pool);
        } catch (JmxException e) {
            log.error(e, "Error exporting memory pool %s", poolId);
        }
    }
    return builder.build();
}

Example 12 with MemoryPoolId

use of com.facebook.presto.spi.memory.MemoryPoolId in project presto by prestodb.

the class TestSqlTask method createInitialTask.

public SqlTask createInitialTask() {
    TaskId taskId = new TaskId("query", 0, 0, nextTaskId.incrementAndGet());
    URI location = URI.create("fake://task/" + taskId);
    QueryContext queryContext = new QueryContext(new QueryId("query"), new DataSize(1, MEGABYTE), new DataSize(2, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, GIGABYTE), new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE)), new TestingGcMonitor(), taskNotificationExecutor, driverYieldExecutor, new DataSize(1, MEGABYTE), new SpillSpaceTracker(new DataSize(1, GIGABYTE)), listJsonCodec(TaskMemoryReservationSummary.class));
    queryContext.addTaskContext(new TaskStateMachine(taskId, taskNotificationExecutor), testSessionBuilder().build(), Optional.of(PLAN_FRAGMENT.getRoot()), false, false, false, false, false);
    return createSqlTask(taskId, location, "fake", queryContext, sqlTaskExecutionFactory, new MockExchangeClientSupplier(), taskNotificationExecutor, Functions.identity(), new DataSize(32, MEGABYTE), new CounterStat(), new SpoolingOutputBufferFactory(new FeaturesConfig()));
}

Example 13 with MemoryPoolId

use of com.facebook.presto.spi.memory.MemoryPoolId in project presto by prestodb.

the class TestMemoryRevokingScheduler method testQueryMemoryRevoking.

@Test
public void testQueryMemoryRevoking() throws Exception {
    // The various tasks created here use a small amount of system memory independent of what's set explicitly
    // in this test. Triggering spilling based on differences of thousands of bytes rather than hundreds
    // makes the test resilient to any noise that creates.
    // There can still be a race condition where some of these allocations are made when the total memory is above
    // the spill threshold, but in revokeMemory() some memory is reduced between when we get the total memory usage
    // and when we get the task memory usage.  This can cause some extra spilling.
    // To prevent flakiness in the test, we reset revoke memory requested for all operators, even if only one spilled.
    QueryId queryId = new QueryId("query");
    // use a larger memory pool so that we don't trigger spilling due to filling the memory pool
    MemoryPool queryLimitMemoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(100, GIGABYTE));
    SqlTask sqlTask1 = newSqlTask(queryId, queryLimitMemoryPool);
    TestOperatorContext operatorContext11 = createTestingOperatorContexts(sqlTask1, "operator11");
    TestOperatorContext operatorContext12 = createTestingOperatorContexts(sqlTask1, "operator12");
    SqlTask sqlTask2 = newSqlTask(queryId, queryLimitMemoryPool);
    TestOperatorContext operatorContext2 = createTestingOperatorContexts(sqlTask2, "operator2");
    allOperatorContexts = ImmutableSet.of(operatorContext11, operatorContext12, operatorContext2);
    List<SqlTask> tasks = ImmutableList.of(sqlTask1, sqlTask2);
    MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(singletonList(queryLimitMemoryPool), () -> tasks, queryContexts::get, 1.0, 1.0, ORDER_BY_REVOCABLE_BYTES, true);
    try {
        scheduler.start();
        assertMemoryRevokingNotRequested();
        operatorContext11.localRevocableMemoryContext().setBytes(150_000);
        operatorContext2.localRevocableMemoryContext().setBytes(100_000);
        // at this point, Task1 = 150k total bytes, Task2 = 100k total bytes
        // this ensures that we are waiting for the memory revocation listener and not using polling-based revoking
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingNotRequested();
        operatorContext12.localRevocableMemoryContext().setBytes(300_000);
        // at this point, Task1 =  450k total bytes, Task2 = 100k total bytes
        scheduler.awaitAsynchronousCallbacksRun();
        // only operator11 should revoke since we need to revoke only 50k bytes
        // limit - (task1 + task2) => 500k - (450k + 100k) = 50k byte to revoke
        assertMemoryRevokingRequestedFor(operatorContext11);
        // revoke all bytes in operator11
        operatorContext11.localRevocableMemoryContext().setBytes(0);
        // at this point, Task1 = 300k total bytes, Task2 = 100k total bytes
        scheduler.awaitAsynchronousCallbacksRun();
        operatorContext11.resetMemoryRevokingRequested();
        operatorContext12.resetMemoryRevokingRequested();
        operatorContext2.resetMemoryRevokingRequested();
        assertMemoryRevokingNotRequested();
        operatorContext11.localRevocableMemoryContext().setBytes(20_000);
        // at this point, Task1 = 320,000 total bytes (oc11 - 20k, oc12 - 300k), Task2 = 100k total bytes
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingNotRequested();
        operatorContext2.localSystemMemoryContext().setBytes(150_000);
        // at this point, Task1 = 320K total bytes, Task2 = 250K total bytes
        // both operator11 and operator 12 are revoking since we revoke in order of operator creation within the task until we are below the memory revoking threshold
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingRequestedFor(operatorContext11, operatorContext12);
        operatorContext11.localRevocableMemoryContext().setBytes(0);
        operatorContext12.localRevocableMemoryContext().setBytes(0);
        scheduler.awaitAsynchronousCallbacksRun();
        operatorContext11.resetMemoryRevokingRequested();
        operatorContext12.resetMemoryRevokingRequested();
        operatorContext2.resetMemoryRevokingRequested();
        assertMemoryRevokingNotRequested();
        operatorContext11.localRevocableMemoryContext().setBytes(50_000);
        operatorContext12.localRevocableMemoryContext().setBytes(50_000);
        operatorContext2.localSystemMemoryContext().setBytes(150_000);
        operatorContext2.localRevocableMemoryContext().setBytes(150_000);
        scheduler.awaitAsynchronousCallbacksRun();
        // no need to revoke
        assertMemoryRevokingNotRequested();
        // at this point, Task1 = 75k total bytes, Task2 = 300k total bytes (150k revocable, 150k system)
        operatorContext12.localUserMemoryContext().setBytes(300_000);
        // at this point, Task1 = 400K total bytes (100k revocable, 300k user), Task2 = 300k total bytes (150k revocable, 150k system)
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingRequestedFor(operatorContext2, operatorContext11);
    } finally {
        scheduler.stop();
    }
}

Example 14 with MemoryPoolId

use of com.facebook.presto.spi.memory.MemoryPoolId in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method assertMemoryPoolMap.

private void assertMemoryPoolMap(ResourceManagerClusterStateProvider provider, int memoryPoolSize, MemoryPoolId memoryPoolId, int assignedQueries, int blockedNodes, int maxBytes, int reservedBytes, int reservedRevocableBytes, Optional<String> largestMemoryQuery) {
    Map<MemoryPoolId, ClusterMemoryPoolInfo> memoryPoolMap = provider.getClusterMemoryPoolInfo();
    assertNotNull(memoryPoolMap);
    assertEquals(memoryPoolMap.size(), memoryPoolSize);
    ClusterMemoryPoolInfo clusterMemoryPoolInfo = memoryPoolMap.get(memoryPoolId);
    assertNotNull(clusterMemoryPoolInfo);
    assertEquals(clusterMemoryPoolInfo.getAssignedQueries(), assignedQueries);
    assertEquals(clusterMemoryPoolInfo.getBlockedNodes(), blockedNodes);
    assertEquals(clusterMemoryPoolInfo.getMemoryPoolInfo().getMaxBytes(), maxBytes);
    assertEquals(clusterMemoryPoolInfo.getMemoryPoolInfo().getReservedBytes(), reservedBytes);
    assertEquals(clusterMemoryPoolInfo.getMemoryPoolInfo().getReservedRevocableBytes(), reservedRevocableBytes);
    assertEquals(clusterMemoryPoolInfo.getLargestMemoryQuery().map(QueryId::getId), largestMemoryQuery);
}

Example 15 with MemoryPoolId

use of com.facebook.presto.spi.memory.MemoryPoolId in project presto by prestodb.

the class TestingTaskContext method createTaskContext.

public static TaskContext createTaskContext(Executor executor, Session session, DataSize maxMemory) {
    MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE));
    MemoryPool systemMemoryPool = new MemoryPool(new MemoryPoolId("testSystem"), new DataSize(1, GIGABYTE));
    QueryContext queryContext = new QueryContext(new QueryId("test_query"), maxMemory, memoryPool, systemMemoryPool, executor);
    return createTaskContext(queryContext, executor, session);
}