Examples with MemoryPoolId com.facebook.presto.spi.memory.MemoryPoolId used on opensource projects

Example 16 with MemoryPoolId

use of com.facebook.presto.spi.memory.MemoryPoolId in project presto by prestodb.

the class AbstractOperatorBenchmark method runOnce.

@Override
protected Map<String, Long> runOnce() {
    Session session = testSessionBuilder().setSystemProperty("optimizer.optimize-hash-generation", "true").build();
    MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE));
    SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(new DataSize(1, GIGABYTE));
    TaskContext taskContext = new QueryContext(new QueryId("test"), new DataSize(256, MEGABYTE), new DataSize(512, MEGABYTE), new DataSize(256, MEGABYTE), new DataSize(1, GIGABYTE), memoryPool, new TestingGcMonitor(), localQueryRunner.getExecutor(), localQueryRunner.getScheduler(), new DataSize(256, MEGABYTE), spillSpaceTracker, listJsonCodec(TaskMemoryReservationSummary.class)).addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0, 0), localQueryRunner.getExecutor()), session, Optional.empty(), false, false, false, false, false);
    CpuTimer cpuTimer = new CpuTimer();
    Map<String, Long> executionStats = execute(taskContext);
    CpuDuration executionTime = cpuTimer.elapsedTime();
    TaskStats taskStats = taskContext.getTaskStats();
    long inputRows = taskStats.getRawInputPositions();
    long inputBytes = taskStats.getRawInputDataSizeInBytes();
    long outputRows = taskStats.getOutputPositions();
    long outputBytes = taskStats.getOutputDataSizeInBytes();
    double inputMegaBytes = new DataSize(inputBytes, BYTE).getValue(MEGABYTE);
    return ImmutableMap.<String, Long>builder().putAll(executionStats).put("elapsed_millis", executionTime.getWall().toMillis()).put("input_rows_per_second", (long) (inputRows / executionTime.getWall().getValue(SECONDS))).put("output_rows_per_second", (long) (outputRows / executionTime.getWall().getValue(SECONDS))).put("input_megabytes", (long) inputMegaBytes).put("input_megabytes_per_second", (long) (inputMegaBytes / executionTime.getWall().getValue(SECONDS))).put("wall_nanos", executionTime.getWall().roundTo(NANOSECONDS)).put("cpu_nanos", executionTime.getCpu().roundTo(NANOSECONDS)).put("user_nanos", executionTime.getUser().roundTo(NANOSECONDS)).put("input_rows", inputRows).put("input_bytes", inputBytes).put("output_rows", outputRows).put("output_bytes", outputBytes).build();
}

Example 17 with MemoryPoolId

use of com.facebook.presto.spi.memory.MemoryPoolId in project presto by prestodb.

the class ResourceManagerClusterStateProvider method getClusterMemoryPoolInfoInternal.

private Map<MemoryPoolId, ClusterMemoryPoolInfo> getClusterMemoryPoolInfoInternal() {
    List<MemoryInfo> memoryInfos = nodeStatuses.values().stream().map(nodeStatus -> nodeStatus.getNodeStatus().getMemoryInfo()).collect(toImmutableList());
    int queriesAssignedToGeneralPool = 0;
    int queriesAssignedToReservedPool = 0;
    Query largestGeneralPoolQuery = null;
    for (CoordinatorQueriesState nodeQueryState : nodeQueryStates.values()) {
        for (Query query : nodeQueryState.getActiveQueries()) {
            MemoryPoolId memoryPool = query.getBasicQueryInfo().getMemoryPool();
            if (GENERAL_POOL.equals(memoryPool)) {
                queriesAssignedToGeneralPool = Math.incrementExact(queriesAssignedToGeneralPool);
                if (!resourceOvercommit(query.getBasicQueryInfo().getSession().toSession(sessionPropertyManager))) {
                    largestGeneralPoolQuery = getLargestMemoryQuery(Optional.ofNullable(largestGeneralPoolQuery), query);
                }
            } else if (RESERVED_POOL.equals(memoryPool)) {
                queriesAssignedToReservedPool = Math.incrementExact(queriesAssignedToReservedPool);
            } else {
                throw new IllegalArgumentException("Unrecognized memory pool: " + memoryPool);
            }
        }
    }
    ImmutableMap.Builder<MemoryPoolId, ClusterMemoryPoolInfo> memoryPoolInfos = ImmutableMap.builder();
    ClusterMemoryPool pool = new ClusterMemoryPool(GENERAL_POOL);
    pool.update(memoryInfos, queriesAssignedToGeneralPool);
    ClusterMemoryPoolInfo clusterInfo = pool.getClusterInfo(Optional.ofNullable(largestGeneralPoolQuery).map(Query::getQueryId));
    memoryPoolInfos.put(GENERAL_POOL, clusterInfo);
    if (isReservedPoolEnabled) {
        pool = new ClusterMemoryPool(RESERVED_POOL);
        pool.update(memoryInfos, queriesAssignedToReservedPool);
        memoryPoolInfos.put(RESERVED_POOL, pool.getClusterInfo());
    }
    return memoryPoolInfos.build();
}

Example 18 with MemoryPoolId

use of com.facebook.presto.spi.memory.MemoryPoolId in project presto by prestodb.

the class TestMemoryRevokingScheduler method testQueryMemoryNotRevokedWhenNotEnabled.

@Test
public void testQueryMemoryNotRevokedWhenNotEnabled() throws Exception {
    // The various tasks created here use a small amount of system memory independent of what's set explicitly
    // in this test. Triggering spilling based on differences of thousands of bytes rather than hundreds
    // makes the test resilient to any noise that creates.
    QueryId queryId = new QueryId("query");
    // use a larger memory pool so that we don't trigger spilling due to filling the memory pool
    MemoryPool queryLimitMemoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(100, GIGABYTE));
    SqlTask sqlTask1 = newSqlTask(queryId, queryLimitMemoryPool);
    TestOperatorContext operatorContext11 = createTestingOperatorContexts(sqlTask1, "operator11");
    allOperatorContexts = ImmutableSet.of(operatorContext11);
    List<SqlTask> tasks = ImmutableList.of(sqlTask1);
    MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(singletonList(queryLimitMemoryPool), () -> tasks, queryContexts::get, 1.0, 1.0, ORDER_BY_REVOCABLE_BYTES, false);
    try {
        scheduler.start();
        assertMemoryRevokingNotRequested();
        // exceed the query memory limit of 500KB
        operatorContext11.localRevocableMemoryContext().setBytes(600_000);
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingNotRequested();
    } finally {
        scheduler.stop();
    }
}

Example 19 with MemoryPoolId

use of com.facebook.presto.spi.memory.MemoryPoolId in project presto by prestodb.

the class TestMemoryRevokingScheduler method testCountAlreadyRevokedMemoryWithinAPool.

@Test
public void testCountAlreadyRevokedMemoryWithinAPool() throws Exception {
    // Given
    MemoryPool anotherMemoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(10, BYTE));
    SqlTask sqlTask1 = newSqlTask(new QueryId("q1"), anotherMemoryPool);
    OperatorContext operatorContext1 = createContexts(sqlTask1);
    SqlTask sqlTask2 = newSqlTask(new QueryId("q2"), memoryPool);
    OperatorContext operatorContext2 = createContexts(sqlTask2);
    List<SqlTask> tasks = ImmutableList.of(sqlTask1, sqlTask2);
    MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(asList(memoryPool, anotherMemoryPool), () -> tasks, queryContexts::get, 1.0, 1.0, ORDER_BY_CREATE_TIME, false);
    try {
        scheduler.start();
        allOperatorContexts = ImmutableSet.of(operatorContext1, operatorContext2);
        /*
             * sqlTask1 fills its pool
             */
        operatorContext1.localRevocableMemoryContext().setBytes(12);
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingRequestedFor(operatorContext1);
        /*
             * When sqlTask2 fills its pool
             */
        operatorContext2.localRevocableMemoryContext().setBytes(12);
        scheduler.awaitAsynchronousCallbacksRun();
        /*
             * Then sqlTask2 should be asked to revoke its memory too
             */
        assertMemoryRevokingRequestedFor(operatorContext1, operatorContext2);
    } finally {
        scheduler.stop();
    }
}

Example 20 with MemoryPoolId

use of com.facebook.presto.spi.memory.MemoryPoolId in project presto by prestodb.

the class LocalMemoryManager method configureMemoryPools.

private void configureMemoryPools(NodeMemoryConfig config, long availableMemory) {
    validateHeapHeadroom(config, availableMemory);
    maxMemory = new DataSize(availableMemory - config.getHeapHeadroom().toBytes(), BYTE);
    checkArgument(config.getMaxQueryMemoryPerNode().toBytes() <= config.getMaxQueryTotalMemoryPerNode().toBytes(), "Max query memory per node (%s) cannot be greater than the max query total memory per node (%s).", QUERY_MAX_MEMORY_PER_NODE_CONFIG, QUERY_MAX_TOTAL_MEMORY_PER_NODE_CONFIG);
    checkArgument(config.getMaxQueryMemoryPerNode().toBytes() >= config.getSoftMaxQueryMemoryPerNode().toBytes(), "Max query memory per node (%s) must be >= soft limit (%s).", QUERY_MAX_MEMORY_PER_NODE_CONFIG, QUERY_SOFT_MAX_MEMORY_PER_NODE_CONFIG);
    checkArgument(config.getMaxQueryTotalMemoryPerNode().toBytes() >= config.getSoftMaxQueryTotalMemoryPerNode().toBytes(), "Max query total memory per node (%s) must be >= soft limit (%s).", QUERY_MAX_TOTAL_MEMORY_PER_NODE_CONFIG, QUERY_SOFT_MAX_TOTAL_MEMORY_PER_NODE_CONFIG);
    ImmutableMap.Builder<MemoryPoolId, MemoryPool> builder = ImmutableMap.builder();
    long generalPoolSize = maxMemory.toBytes();
    if (config.isReservedPoolEnabled()) {
        builder.put(RESERVED_POOL, new MemoryPool(RESERVED_POOL, config.getMaxQueryTotalMemoryPerNode()));
        generalPoolSize -= config.getMaxQueryTotalMemoryPerNode().toBytes();
    }
    verify(generalPoolSize > 0, "general memory pool size is 0");
    builder.put(GENERAL_POOL, new MemoryPool(GENERAL_POOL, new DataSize(generalPoolSize, BYTE)));
    this.pools = builder.build();
}