Examples with MemoryPool com.facebook.presto.memory.MemoryPool used on opensource projects

Example 6 with MemoryPool

use of com.facebook.presto.memory.MemoryPool in project presto by prestodb.

the class TestingTaskContext method createTaskContext.

public static TaskContext createTaskContext(Executor executor, Session session, DataSize maxMemory) {
    MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE));
    MemoryPool systemMemoryPool = new MemoryPool(new MemoryPoolId("testSystem"), new DataSize(1, GIGABYTE));
    QueryContext queryContext = new QueryContext(new QueryId("test_query"), maxMemory, memoryPool, systemMemoryPool, executor);
    return createTaskContext(queryContext, executor, session);
}

Example 7 with MemoryPool

use of com.facebook.presto.memory.MemoryPool in project presto by prestodb.

the class AbstractOperatorBenchmark method runOnce.

@Override
protected Map<String, Long> runOnce() {
    Session session = testSessionBuilder().setSystemProperty("optimizer.optimize-hash-generation", "true").build();
    MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE));
    SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(new DataSize(1, GIGABYTE));
    TaskContext taskContext = new QueryContext(new QueryId("test"), new DataSize(256, MEGABYTE), new DataSize(512, MEGABYTE), new DataSize(256, MEGABYTE), new DataSize(1, GIGABYTE), memoryPool, new TestingGcMonitor(), localQueryRunner.getExecutor(), localQueryRunner.getScheduler(), new DataSize(256, MEGABYTE), spillSpaceTracker, listJsonCodec(TaskMemoryReservationSummary.class)).addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0, 0), localQueryRunner.getExecutor()), session, Optional.empty(), false, false, false, false, false);
    CpuTimer cpuTimer = new CpuTimer();
    Map<String, Long> executionStats = execute(taskContext);
    CpuDuration executionTime = cpuTimer.elapsedTime();
    TaskStats taskStats = taskContext.getTaskStats();
    long inputRows = taskStats.getRawInputPositions();
    long inputBytes = taskStats.getRawInputDataSizeInBytes();
    long outputRows = taskStats.getOutputPositions();
    long outputBytes = taskStats.getOutputDataSizeInBytes();
    double inputMegaBytes = new DataSize(inputBytes, BYTE).getValue(MEGABYTE);
    return ImmutableMap.<String, Long>builder().putAll(executionStats).put("elapsed_millis", executionTime.getWall().toMillis()).put("input_rows_per_second", (long) (inputRows / executionTime.getWall().getValue(SECONDS))).put("output_rows_per_second", (long) (outputRows / executionTime.getWall().getValue(SECONDS))).put("input_megabytes", (long) inputMegaBytes).put("input_megabytes_per_second", (long) (inputMegaBytes / executionTime.getWall().getValue(SECONDS))).put("wall_nanos", executionTime.getWall().roundTo(NANOSECONDS)).put("cpu_nanos", executionTime.getCpu().roundTo(NANOSECONDS)).put("user_nanos", executionTime.getUser().roundTo(NANOSECONDS)).put("input_rows", inputRows).put("input_bytes", inputBytes).put("output_rows", outputRows).put("output_bytes", outputBytes).build();
}

Example 8 with MemoryPool

use of com.facebook.presto.memory.MemoryPool in project presto by prestodb.

the class SqlTaskManager method updateMemoryPoolAssignments.

@Override
public synchronized void updateMemoryPoolAssignments(MemoryPoolAssignmentsRequest assignments) {
    String assignmentCoordinatorId = assignments.getCoordinatorId();
    long assignmentVersion = assignments.getVersion();
    if (assignmentVersion <= currentMemoryPoolAssignmentVersions.getOrDefault(assignmentCoordinatorId, Long.MIN_VALUE)) {
        return;
    }
    currentMemoryPoolAssignmentVersions.put(assignmentCoordinatorId, assignmentVersion);
    for (MemoryPoolAssignment assignment : assignments.getAssignments()) {
        if (assignment.getPoolId().equals(GENERAL_POOL)) {
            queryContexts.getUnchecked(assignment.getQueryId()).setMemoryPool(localMemoryManager.getGeneralPool());
        } else if (assignment.getPoolId().equals(RESERVED_POOL)) {
            MemoryPool reservedPool = localMemoryManager.getReservedPool().orElseThrow(() -> new IllegalArgumentException(format("Cannot move %s to the reserved pool as the reserved pool is not enabled", assignment.getQueryId())));
            queryContexts.getUnchecked(assignment.getQueryId()).setMemoryPool(reservedPool);
        } else {
            throw new IllegalArgumentException(format("Cannot move %s to %s as the target memory pool id is invalid", assignment.getQueryId(), assignment.getPoolId()));
        }
    }
}

Example 9 with MemoryPool

use of com.facebook.presto.memory.MemoryPool in project presto by prestodb.

the class MemoryRevokingScheduler method revokeQueryMemory.

private void revokeQueryMemory(QueryContext queryContext, long maxTotalMemory) {
    QueryId queryId = queryContext.getQueryId();
    MemoryPool memoryPool = queryContext.getMemoryPool();
    // get a fresh value for queryTotalMemory in case it's changed (e.g. by a previous revocation request)
    long queryTotalMemory = getTotalQueryMemoryReservation(queryId, memoryPool);
    // order tasks by decreasing revocableMemory so that we don't spill more tasks than needed
    SortedMap<Long, TaskContext> queryTaskContextsMap = new TreeMap<>(Comparator.reverseOrder());
    queryContext.getAllTaskContexts().forEach(taskContext -> queryTaskContextsMap.put(taskContext.getTaskMemoryContext().getRevocableMemory(), taskContext));
    AtomicLong remainingBytesToRevoke = new AtomicLong(queryTotalMemory - maxTotalMemory);
    Collection<TaskContext> queryTaskContexts = queryTaskContextsMap.values();
    remainingBytesToRevoke.addAndGet(-MemoryRevokingSchedulerUtils.getMemoryAlreadyBeingRevoked(queryTaskContexts, remainingBytesToRevoke.get()));
    for (TaskContext taskContext : queryTaskContexts) {
        if (remainingBytesToRevoke.get() <= 0) {
            break;
        }
        taskContext.accept(new VoidTraversingQueryContextVisitor<AtomicLong>() {

            @Override
            public Void visitOperatorContext(OperatorContext operatorContext, AtomicLong remainingBytesToRevoke) {
                if (remainingBytesToRevoke.get() > 0) {
                    long revokedBytes = operatorContext.requestMemoryRevoking();
                    if (revokedBytes > 0) {
                        remainingBytesToRevoke.addAndGet(-revokedBytes);
                        log.debug("taskId=%s: requested revoking %s; remaining %s", taskContext.getTaskId(), revokedBytes, remainingBytesToRevoke);
                    }
                }
                return null;
            }
        }, remainingBytesToRevoke);
    }
}

Example 10 with MemoryPool

use of com.facebook.presto.memory.MemoryPool in project presto by prestodb.

the class TestMemoryRevokingScheduler method setUp.

@BeforeMethod
public void setUp() {
    memoryPool = new MemoryPool(GENERAL_POOL, new DataSize(10, BYTE));
    TaskExecutor taskExecutor = new TaskExecutor(8, 16, 3, 4, TASK_FAIR, Ticker.systemTicker());
    taskExecutor.start();
    // Must be single threaded
    singleThreadedExecutor = newSingleThreadExecutor(threadsNamed("task-notification-%s"));
    singleThreadedScheduledExecutor = newScheduledThreadPool(1, threadsNamed("task-notification-%s"));
    scheduledExecutor = newScheduledThreadPool(2, threadsNamed("task-notification-%s"));
    LocalExecutionPlanner planner = createTestingPlanner();
    sqlTaskExecutionFactory = new SqlTaskExecutionFactory(singleThreadedExecutor, taskExecutor, planner, new BlockEncodingManager(), new OrderingCompiler(), createTestSplitMonitor(), new TaskManagerConfig().setPerOperatorAllocationTrackingEnabled(true).setTaskCpuTimerEnabled(true).setPerOperatorAllocationTrackingEnabled(true).setTaskAllocationTrackingEnabled(true));
    allOperatorContexts = null;
    TestOperatorContext.firstOperator = null;
}