Examples with TaskStateMachine io.prestosql.execution.TaskStateMachine used on opensource projects

Example 1 with TaskStateMachine

use of io.prestosql.execution.TaskStateMachine in project hetu-core by openlookeng.

the class TestMemoryTracking method setUpTest.

@BeforeMethod
public void setUpTest() {
    memoryPool = new MemoryPool(new MemoryPoolId("test"), memoryPoolSize);
    queryContext = new QueryContext(new QueryId("test_query"), queryMaxMemory, queryMaxTotalMemory, memoryPool, new TestingGcMonitor(), notificationExecutor, yieldExecutor, queryMaxSpillSize, spillSpaceTracker, NOOP_SNAPSHOT_UTILS);
    taskContext = queryContext.addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0), notificationExecutor), testSessionBuilder().build(), true, true, OptionalInt.empty(), Optional.empty(), TESTING_SERDE_FACTORY);
    pipelineContext = taskContext.addPipelineContext(0, true, true, false);
    driverContext = pipelineContext.addDriverContext();
    operatorContext = driverContext.addOperatorContext(1, new PlanNodeId("a"), "test-operator");
}

Example 2 with TaskStateMachine

use of io.prestosql.execution.TaskStateMachine in project hetu-core by openlookeng.

the class MemoryLocalQueryRunner method execute.

public List<Page> execute(@Language("SQL") String query) {
    MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(2, GIGABYTE));
    SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(new DataSize(1, GIGABYTE));
    QueryContext queryContext = new QueryContext(new QueryId("test"), new DataSize(1, GIGABYTE), new DataSize(2, GIGABYTE), memoryPool, new TestingGcMonitor(), localQueryRunner.getExecutor(), localQueryRunner.getScheduler(), new DataSize(4, GIGABYTE), spillSpaceTracker, NOOP_SNAPSHOT_UTILS);
    TaskContext taskContext = queryContext.addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0), localQueryRunner.getExecutor()), localQueryRunner.getDefaultSession(), false, false, OptionalInt.empty(), Optional.empty(), TESTING_SERDE_FACTORY);
    // Use NullOutputFactory to avoid coping out results to avoid affecting benchmark results
    ImmutableList.Builder<Page> output = ImmutableList.builder();
    List<Driver> drivers = localQueryRunner.createDrivers(query, new PageConsumerOperator.PageConsumerOutputFactory(types -> output::add), taskContext);
    boolean done = false;
    while (!done) {
        boolean processed = false;
        for (Driver driver : drivers) {
            if (!driver.isFinished()) {
                driver.process();
                processed = true;
            }
        }
        done = !processed;
    }
    return output.build();
}

Example 3 with TaskStateMachine

use of io.prestosql.execution.TaskStateMachine in project hetu-core by openlookeng.

the class BufferTestUtils method newTestingTaskContext.

static TaskContext newTestingTaskContext() {
    ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
    QueryContext queryContext = new QueryContext(new QueryId("query_id"), new DataSize(1, MEGABYTE), new DataSize(2, MEGABYTE), new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE)), new TestingGcMonitor(), executor, executor, new DataSize(1, MEGABYTE), new SpillSpaceTracker(new DataSize(1, GIGABYTE)), NOOP_SNAPSHOT_UTILS);
    TaskId taskId = TaskId.valueOf("query_id.1.2");
    TaskContext taskContext = queryContext.addTaskContext(new TaskStateMachine(taskId, executor), TEST_SNAPSHOT_SESSION, false, false, OptionalInt.empty(), Optional.empty(), TESTING_SERDE_FACTORY);
    taskContext.getSnapshotManager().setTotalComponents(100);
    return taskContext;
}

Example 4 with TaskStateMachine

use of io.prestosql.execution.TaskStateMachine in project hetu-core by openlookeng.

the class TestQueryContext method testMoveTaggedAllocations.

@Test
public void testMoveTaggedAllocations() {
    MemoryPool generalPool = new MemoryPool(GENERAL_POOL, new DataSize(10_000, BYTE));
    MemoryPool reservedPool = new MemoryPool(RESERVED_POOL, new DataSize(10_000, BYTE));
    QueryId queryId = new QueryId("query");
    QueryContext queryContext = createQueryContext(queryId, generalPool);
    TaskStateMachine taskStateMachine = new TaskStateMachine(TaskId.valueOf("task-id"), TEST_EXECUTOR);
    TaskContext taskContext = queryContext.addTaskContext(taskStateMachine, TEST_SESSION, false, false, OptionalInt.empty(), Optional.empty(), TESTING_SERDE_FACTORY);
    DriverContext driverContext = taskContext.addPipelineContext(0, false, false, false).addDriverContext();
    OperatorContext operatorContext = driverContext.addOperatorContext(0, new PlanNodeId("test"), "test");
    // allocate some memory in the general pool
    LocalMemoryContext memoryContext = operatorContext.aggregateUserMemoryContext().newLocalMemoryContext("test_context");
    memoryContext.setBytes(1_000);
    Map<String, Long> allocations = generalPool.getTaggedMemoryAllocations().get(queryId);
    assertEquals(allocations, ImmutableMap.of("test_context", 1_000L));
    queryContext.setMemoryPool(reservedPool);
    assertNull(generalPool.getTaggedMemoryAllocations().get(queryId));
    allocations = reservedPool.getTaggedMemoryAllocations().get(queryId);
    assertEquals(allocations, ImmutableMap.of("test_context", 1_000L));
    assertEquals(generalPool.getFreeBytes(), 10_000);
    assertEquals(reservedPool.getFreeBytes(), 9_000);
    memoryContext.close();
    assertEquals(generalPool.getFreeBytes(), 10_000);
    assertEquals(reservedPool.getFreeBytes(), 10_000);
}

Example 5 with TaskStateMachine

use of io.prestosql.execution.TaskStateMachine in project hetu-core by openlookeng.

the class AbstractOperatorBenchmark method runOnce.

@Override
protected Map<String, Long> runOnce() {
    Session setSession = testSessionBuilder().setSystemProperty("optimizer.optimize-hash-generation", "true").build();
    MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE));
    SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(new DataSize(1, GIGABYTE));
    TaskContext taskContext = new QueryContext(new QueryId("test"), new DataSize(256, MEGABYTE), new DataSize(512, MEGABYTE), memoryPool, new TestingGcMonitor(), localQueryRunner.getExecutor(), localQueryRunner.getScheduler(), new DataSize(256, MEGABYTE), spillSpaceTracker, NOOP_SNAPSHOT_UTILS).addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0), localQueryRunner.getExecutor()), setSession, false, false, OptionalInt.empty(), Optional.empty(), TESTING_SERDE_FACTORY);
    CpuTimer cpuTimer = new CpuTimer();
    Map<String, Long> executionStats = execute(taskContext);
    CpuDuration executionTime = cpuTimer.elapsedTime();
    TaskStats taskStats = taskContext.getTaskStats();
    long inputRows = taskStats.getRawInputPositions();
    long inputBytes = taskStats.getRawInputDataSize().toBytes();
    long outputRows = taskStats.getOutputPositions();
    long outputBytes = taskStats.getOutputDataSize().toBytes();
    double inputMegaBytes = new DataSize(inputBytes, BYTE).getValue(MEGABYTE);
    return ImmutableMap.<String, Long>builder().putAll(executionStats).put("elapsed_millis", executionTime.getWall().toMillis()).put("input_rows_per_second", (long) (inputRows / executionTime.getWall().getValue(SECONDS))).put("output_rows_per_second", (long) (outputRows / executionTime.getWall().getValue(SECONDS))).put("input_megabytes", (long) inputMegaBytes).put("input_megabytes_per_second", (long) (inputMegaBytes / executionTime.getWall().getValue(SECONDS))).put("wall_nanos", executionTime.getWall().roundTo(NANOSECONDS)).put("cpu_nanos", executionTime.getCpu().roundTo(NANOSECONDS)).put("user_nanos", executionTime.getUser().roundTo(NANOSECONDS)).put("input_rows", inputRows).put("input_bytes", inputBytes).put("output_rows", outputRows).put("output_bytes", outputBytes).build();
}