Search in sources :

Example 1 with MemoryPool

use of io.trino.memory.MemoryPool in project trino by trinodb.

the class TestSqlTask method createInitialTask.

private SqlTask createInitialTask() {
    TaskId taskId = new TaskId(new StageId("query", 0), nextTaskId.incrementAndGet(), 0);
    URI location = URI.create("fake://task/" + taskId);
    QueryContext queryContext = new QueryContext(new QueryId("query"), DataSize.of(1, MEGABYTE), new MemoryPool(DataSize.of(1, GIGABYTE)), new TestingGcMonitor(), taskNotificationExecutor, driverYieldExecutor, DataSize.of(1, MEGABYTE), new SpillSpaceTracker(DataSize.of(1, GIGABYTE)));
    queryContext.addTaskContext(new TaskStateMachine(taskId, taskNotificationExecutor), testSessionBuilder().build(), () -> {
    }, false, false);
    return createSqlTask(taskId, location, "fake", queryContext, sqlTaskExecutionFactory, taskNotificationExecutor, sqlTask -> {
    }, DataSize.of(32, MEGABYTE), DataSize.of(200, MEGABYTE), new ExchangeManagerRegistry(new ExchangeHandleResolver()), new CounterStat());
}
Also used : SpillSpaceTracker(io.trino.spiller.SpillSpaceTracker) CounterStat(io.airlift.stats.CounterStat) QueryId(io.trino.spi.QueryId) TestingGcMonitor(io.airlift.stats.TestingGcMonitor) QueryContext(io.trino.memory.QueryContext) URI(java.net.URI) ExchangeManagerRegistry(io.trino.exchange.ExchangeManagerRegistry) ExchangeHandleResolver(io.trino.metadata.ExchangeHandleResolver) MemoryPool(io.trino.memory.MemoryPool)

Example 2 with MemoryPool

use of io.trino.memory.MemoryPool in project trino by trinodb.

the class TestMemoryRevokingScheduler method setUp.

@BeforeMethod
public void setUp() {
    memoryPool = new MemoryPool(DataSize.ofBytes(10));
    TaskExecutor taskExecutor = new TaskExecutor(8, 16, 3, 4, Ticker.systemTicker());
    taskExecutor.start();
    // Must be single threaded
    executor = newScheduledThreadPool(1, threadsNamed("task-notification-%s"));
    scheduledExecutor = newScheduledThreadPool(2, threadsNamed("task-notification-%s"));
    LocalExecutionPlanner planner = createTestingPlanner();
    sqlTaskExecutionFactory = new SqlTaskExecutionFactory(executor, taskExecutor, planner, createTestSplitMonitor(), new TaskManagerConfig());
    allOperatorContexts = null;
}
Also used : TaskExecutor(io.trino.execution.executor.TaskExecutor) LocalExecutionPlanner(io.trino.sql.planner.LocalExecutionPlanner) MemoryPool(io.trino.memory.MemoryPool) BeforeMethod(org.testng.annotations.BeforeMethod)

Example 3 with MemoryPool

use of io.trino.memory.MemoryPool in project trino by trinodb.

the class GroupByHashYieldAssertion method finishOperatorWithYieldingGroupByHash.

/**
 * @param operatorFactory creates an Operator that should directly or indirectly contain GroupByHash
 * @param getHashCapacity returns the hash table capacity for the input operator
 * @param additionalMemoryInBytes the memory used in addition to the GroupByHash in the operator (e.g., aggregator)
 */
public static GroupByHashYieldResult finishOperatorWithYieldingGroupByHash(List<Page> input, Type hashKeyType, OperatorFactory operatorFactory, Function<Operator, Integer> getHashCapacity, long additionalMemoryInBytes) {
    assertLessThan(additionalMemoryInBytes, 1L << 21, "additionalMemoryInBytes should be a relatively small number");
    List<Page> result = new LinkedList<>();
    // mock an adjustable memory pool
    QueryId queryId = new QueryId("test_query");
    MemoryPool memoryPool = new MemoryPool(DataSize.of(1, GIGABYTE));
    QueryContext queryContext = new QueryContext(queryId, DataSize.of(512, MEGABYTE), memoryPool, new TestingGcMonitor(), EXECUTOR, SCHEDULED_EXECUTOR, DataSize.of(512, MEGABYTE), new SpillSpaceTracker(DataSize.of(512, MEGABYTE)));
    DriverContext driverContext = createTaskContext(queryContext, EXECUTOR, TEST_SESSION).addPipelineContext(0, true, true, false).addDriverContext();
    Operator operator = operatorFactory.createOperator(driverContext);
    // run operator
    int yieldCount = 0;
    long expectedReservedExtraBytes = 0;
    for (Page page : input) {
        // unblocked
        assertTrue(operator.needsInput());
        // saturate the pool with a tiny memory left
        long reservedMemoryInBytes = memoryPool.getFreeBytes() - additionalMemoryInBytes;
        memoryPool.reserve(queryId, "test", reservedMemoryInBytes);
        long oldMemoryUsage = operator.getOperatorContext().getDriverContext().getMemoryUsage();
        int oldCapacity = getHashCapacity.apply(operator);
        // add a page and verify different behaviors
        operator.addInput(page);
        // get output to consume the input
        Page output = operator.getOutput();
        if (output != null) {
            result.add(output);
        }
        long newMemoryUsage = operator.getOperatorContext().getDriverContext().getMemoryUsage();
        // between rehash and memory used by aggregator
        if (newMemoryUsage < DataSize.of(4, MEGABYTE).toBytes()) {
            // free the pool for the next iteration
            memoryPool.free(queryId, "test", reservedMemoryInBytes);
            // this required in case input is blocked
            operator.getOutput();
            continue;
        }
        long actualIncreasedMemory = newMemoryUsage - oldMemoryUsage;
        if (operator.needsInput()) {
            // We have successfully added a page
            // Assert we are not blocked
            assertTrue(operator.getOperatorContext().isWaitingForMemory().isDone());
            // assert the hash capacity is not changed; otherwise, we should have yielded
            assertTrue(oldCapacity == getHashCapacity.apply(operator));
            // We are not going to rehash; therefore, assert the memory increase only comes from the aggregator
            assertLessThan(actualIncreasedMemory, additionalMemoryInBytes);
            // free the pool for the next iteration
            memoryPool.free(queryId, "test", reservedMemoryInBytes);
        } else {
            // We failed to finish the page processing i.e. we yielded
            yieldCount++;
            // Assert we are blocked
            assertFalse(operator.getOperatorContext().isWaitingForMemory().isDone());
            // Hash table capacity should not change
            assertEquals(oldCapacity, (long) getHashCapacity.apply(operator));
            // Increased memory is no smaller than the hash table size and no greater than the hash table size + the memory used by aggregator
            if (hashKeyType == BIGINT) {
                // groupIds and values double by hashCapacity; while valuesByGroupId double by maxFill = hashCapacity / 0.75
                expectedReservedExtraBytes = oldCapacity * (long) (Long.BYTES * 1.75 + Integer.BYTES) + page.getRetainedSizeInBytes();
            } else {
                // groupAddressByHash, groupIdsByHash, and rawHashByHashPosition double by hashCapacity; while groupAddressByGroupId double by maxFill = hashCapacity / 0.75
                expectedReservedExtraBytes = oldCapacity * (long) (Long.BYTES * 1.75 + Integer.BYTES + Byte.BYTES) + page.getRetainedSizeInBytes();
            }
            assertBetweenInclusive(actualIncreasedMemory, expectedReservedExtraBytes, expectedReservedExtraBytes + additionalMemoryInBytes);
            // Output should be blocked as well
            assertNull(operator.getOutput());
            // Free the pool to unblock
            memoryPool.free(queryId, "test", reservedMemoryInBytes);
            // Trigger a process through getOutput() or needsInput()
            output = operator.getOutput();
            if (output != null) {
                result.add(output);
            }
            assertTrue(operator.needsInput());
            // Hash table capacity has increased
            assertGreaterThan(getHashCapacity.apply(operator), oldCapacity);
            // Assert the estimated reserved memory before rehash is very close to the one after rehash
            long rehashedMemoryUsage = operator.getOperatorContext().getDriverContext().getMemoryUsage();
            assertBetweenInclusive(rehashedMemoryUsage * 1.0 / newMemoryUsage, 0.99, 1.01);
            // unblocked
            assertTrue(operator.needsInput());
        }
    }
    result.addAll(finishOperator(operator));
    return new GroupByHashYieldResult(yieldCount, expectedReservedExtraBytes, result);
}
Also used : OperatorAssertion.finishOperator(io.trino.operator.OperatorAssertion.finishOperator) SpillSpaceTracker(io.trino.spiller.SpillSpaceTracker) QueryId(io.trino.spi.QueryId) Page(io.trino.spi.Page) QueryContext(io.trino.memory.QueryContext) LinkedList(java.util.LinkedList) TestingGcMonitor(io.airlift.stats.TestingGcMonitor) MemoryPool(io.trino.memory.MemoryPool)

Example 4 with MemoryPool

use of io.trino.memory.MemoryPool in project trino by trinodb.

the class AbstractOperatorBenchmark method runOnce.

@Override
protected Map<String, Long> runOnce() {
    Session session = testSessionBuilder().setSystemProperty("optimizer.optimize-hash-generation", "true").setTransactionId(this.session.getRequiredTransactionId()).build();
    MemoryPool memoryPool = new MemoryPool(DataSize.of(1, GIGABYTE));
    SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(DataSize.of(1, GIGABYTE));
    TaskContext taskContext = new QueryContext(new QueryId("test"), DataSize.of(256, MEGABYTE), memoryPool, new TestingGcMonitor(), localQueryRunner.getExecutor(), localQueryRunner.getScheduler(), DataSize.of(256, MEGABYTE), spillSpaceTracker).addTaskContext(new TaskStateMachine(new TaskId(new StageId("query", 0), 0, 0), localQueryRunner.getExecutor()), session, () -> {
    }, false, false);
    CpuTimer cpuTimer = new CpuTimer();
    Map<String, Long> executionStats = execute(taskContext);
    CpuDuration executionTime = cpuTimer.elapsedTime();
    TaskStats taskStats = taskContext.getTaskStats();
    long inputRows = taskStats.getRawInputPositions();
    long inputBytes = taskStats.getRawInputDataSize().toBytes();
    long outputRows = taskStats.getOutputPositions();
    long outputBytes = taskStats.getOutputDataSize().toBytes();
    double inputMegaBytes = ((double) inputBytes) / MEGABYTE.inBytes();
    return ImmutableMap.<String, Long>builder().putAll(executionStats).put("elapsed_millis", executionTime.getWall().toMillis()).put("input_rows_per_second", (long) (inputRows / executionTime.getWall().getValue(SECONDS))).put("output_rows_per_second", (long) (outputRows / executionTime.getWall().getValue(SECONDS))).put("input_megabytes", (long) inputMegaBytes).put("input_megabytes_per_second", (long) (inputMegaBytes / executionTime.getWall().getValue(SECONDS))).put("wall_nanos", executionTime.getWall().roundTo(NANOSECONDS)).put("cpu_nanos", executionTime.getCpu().roundTo(NANOSECONDS)).put("user_nanos", executionTime.getUser().roundTo(NANOSECONDS)).put("input_rows", inputRows).put("input_bytes", inputBytes).put("output_rows", outputRows).put("output_bytes", outputBytes).buildOrThrow();
}
Also used : SpillSpaceTracker(io.trino.spiller.SpillSpaceTracker) TaskContext(io.trino.operator.TaskContext) TaskId(io.trino.execution.TaskId) QueryId(io.trino.spi.QueryId) StageId(io.trino.execution.StageId) QueryContext(io.trino.memory.QueryContext) TaskStats(io.trino.operator.TaskStats) TaskStateMachine(io.trino.execution.TaskStateMachine) TestingGcMonitor(io.airlift.stats.TestingGcMonitor) CpuTimer(io.airlift.stats.CpuTimer) CpuDuration(io.airlift.stats.CpuTimer.CpuDuration) Session(io.trino.Session) MemoryPool(io.trino.memory.MemoryPool)

Example 5 with MemoryPool

use of io.trino.memory.MemoryPool in project trino by trinodb.

the class MemoryLocalQueryRunner method execute.

public List<Page> execute(@Language("SQL") String query) {
    MemoryPool memoryPool = new MemoryPool(DataSize.of(2, GIGABYTE));
    SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(DataSize.of(1, GIGABYTE));
    QueryContext queryContext = new QueryContext(new QueryId("test"), DataSize.of(1, GIGABYTE), memoryPool, new TestingGcMonitor(), localQueryRunner.getExecutor(), localQueryRunner.getScheduler(), DataSize.of(4, GIGABYTE), spillSpaceTracker);
    TaskContext taskContext = queryContext.addTaskContext(new TaskStateMachine(new TaskId(new StageId("query", 0), 0, 0), localQueryRunner.getExecutor()), localQueryRunner.getDefaultSession(), () -> {
    }, false, false);
    // Use NullOutputFactory to avoid coping out results to avoid affecting benchmark results
    ImmutableList.Builder<Page> output = ImmutableList.builder();
    List<Driver> drivers = localQueryRunner.createDrivers(query, new PageConsumerOperator.PageConsumerOutputFactory(types -> output::add), taskContext);
    boolean done = false;
    while (!done) {
        boolean processed = false;
        for (Driver driver : drivers) {
            if (!driver.isFinished()) {
                driver.process();
                processed = true;
            }
        }
        done = !processed;
    }
    return output.build();
}
Also used : QueryId(io.trino.spi.QueryId) SpillSpaceTracker(io.trino.spiller.SpillSpaceTracker) StageId(io.trino.execution.StageId) MemoryConnectorFactory(io.trino.plugin.memory.MemoryConnectorFactory) Page(io.trino.spi.Page) TaskStateMachine(io.trino.execution.TaskStateMachine) PageConsumerOperator(io.trino.testing.PageConsumerOperator) GIGABYTE(io.airlift.units.DataSize.Unit.GIGABYTE) ImmutableList(com.google.common.collect.ImmutableList) TpchConnectorFactory(io.trino.plugin.tpch.TpchConnectorFactory) LocalQueryRunner(io.trino.testing.LocalQueryRunner) Map(java.util.Map) TaskContext(io.trino.operator.TaskContext) MemoryPool(io.trino.memory.MemoryPool) ImmutableMap(com.google.common.collect.ImmutableMap) Language(org.intellij.lang.annotations.Language) QueryContext(io.trino.memory.QueryContext) TaskId(io.trino.execution.TaskId) DataSize(io.airlift.units.DataSize) List(java.util.List) TestingSession.testSessionBuilder(io.trino.testing.TestingSession.testSessionBuilder) TableHandle(io.trino.metadata.TableHandle) QualifiedObjectName(io.trino.metadata.QualifiedObjectName) Metadata(io.trino.metadata.Metadata) Optional(java.util.Optional) Assert.assertTrue(org.testng.Assert.assertTrue) Driver(io.trino.operator.Driver) TestingGcMonitor(io.airlift.stats.TestingGcMonitor) Session(io.trino.Session) SpillSpaceTracker(io.trino.spiller.SpillSpaceTracker) TaskContext(io.trino.operator.TaskContext) TaskId(io.trino.execution.TaskId) ImmutableList(com.google.common.collect.ImmutableList) QueryId(io.trino.spi.QueryId) StageId(io.trino.execution.StageId) Driver(io.trino.operator.Driver) Page(io.trino.spi.Page) QueryContext(io.trino.memory.QueryContext) TaskStateMachine(io.trino.execution.TaskStateMachine) PageConsumerOperator(io.trino.testing.PageConsumerOperator) TestingGcMonitor(io.airlift.stats.TestingGcMonitor) MemoryPool(io.trino.memory.MemoryPool)

Aggregations

MemoryPool (io.trino.memory.MemoryPool)5 TestingGcMonitor (io.airlift.stats.TestingGcMonitor)4 QueryContext (io.trino.memory.QueryContext)4 QueryId (io.trino.spi.QueryId)4 SpillSpaceTracker (io.trino.spiller.SpillSpaceTracker)4 Session (io.trino.Session)2 StageId (io.trino.execution.StageId)2 TaskId (io.trino.execution.TaskId)2 TaskStateMachine (io.trino.execution.TaskStateMachine)2 TaskContext (io.trino.operator.TaskContext)2 Page (io.trino.spi.Page)2 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 CounterStat (io.airlift.stats.CounterStat)1 CpuTimer (io.airlift.stats.CpuTimer)1 CpuDuration (io.airlift.stats.CpuTimer.CpuDuration)1 DataSize (io.airlift.units.DataSize)1 GIGABYTE (io.airlift.units.DataSize.Unit.GIGABYTE)1 ExchangeManagerRegistry (io.trino.exchange.ExchangeManagerRegistry)1 TaskExecutor (io.trino.execution.executor.TaskExecutor)1