Examples with QueryId com.facebook.presto.spi.QueryId used on opensource projects

Example 56 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class TestSqlTask method createInitialTask.

public SqlTask createInitialTask() {
    TaskId taskId = new TaskId("query", 0, 0, nextTaskId.incrementAndGet());
    URI location = URI.create("fake://task/" + taskId);
    QueryContext queryContext = new QueryContext(new QueryId("query"), new DataSize(1, MEGABYTE), new DataSize(2, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, GIGABYTE), new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE)), new TestingGcMonitor(), taskNotificationExecutor, driverYieldExecutor, new DataSize(1, MEGABYTE), new SpillSpaceTracker(new DataSize(1, GIGABYTE)), listJsonCodec(TaskMemoryReservationSummary.class));
    queryContext.addTaskContext(new TaskStateMachine(taskId, taskNotificationExecutor), testSessionBuilder().build(), Optional.of(PLAN_FRAGMENT.getRoot()), false, false, false, false, false);
    return createSqlTask(taskId, location, "fake", queryContext, sqlTaskExecutionFactory, new MockExchangeClientSupplier(), taskNotificationExecutor, Functions.identity(), new DataSize(32, MEGABYTE), new CounterStat(), new SpoolingOutputBufferFactory(new FeaturesConfig()));
}

Example 57 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class TestMemoryRevokingScheduler method testMemoryPoolRevoking.

@Test
public void testMemoryPoolRevoking() throws Exception {
    QueryContext q1 = getOrCreateQueryContext(new QueryId("q1"), memoryPool);
    QueryContext q2 = getOrCreateQueryContext(new QueryId("q2"), memoryPool);
    SqlTask sqlTask1 = newSqlTask(q1.getQueryId(), memoryPool);
    SqlTask sqlTask2 = newSqlTask(q2.getQueryId(), memoryPool);
    TaskContext taskContext1 = getOrCreateTaskContext(sqlTask1);
    PipelineContext pipelineContext11 = taskContext1.addPipelineContext(0, false, false, false);
    DriverContext driverContext111 = pipelineContext11.addDriverContext();
    OperatorContext operatorContext1 = driverContext111.addOperatorContext(1, new PlanNodeId("na"), "na");
    OperatorContext operatorContext2 = driverContext111.addOperatorContext(2, new PlanNodeId("na"), "na");
    DriverContext driverContext112 = pipelineContext11.addDriverContext();
    OperatorContext operatorContext3 = driverContext112.addOperatorContext(3, new PlanNodeId("na"), "na");
    TaskContext taskContext2 = getOrCreateTaskContext(sqlTask2);
    PipelineContext pipelineContext21 = taskContext2.addPipelineContext(1, false, false, false);
    DriverContext driverContext211 = pipelineContext21.addDriverContext();
    OperatorContext operatorContext4 = driverContext211.addOperatorContext(4, new PlanNodeId("na"), "na");
    OperatorContext operatorContext5 = driverContext211.addOperatorContext(5, new PlanNodeId("na"), "na");
    List<SqlTask> tasks = ImmutableList.of(sqlTask1, sqlTask2);
    MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(singletonList(memoryPool), () -> tasks, queryContexts::get, 1.0, 1.0, ORDER_BY_CREATE_TIME, false);
    try {
        scheduler.start();
        allOperatorContexts = ImmutableSet.of(operatorContext1, operatorContext2, operatorContext3, operatorContext4, operatorContext5);
        assertMemoryRevokingNotRequested();
        assertEquals(10, memoryPool.getFreeBytes());
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingNotRequested();
        LocalMemoryContext revocableMemory1 = operatorContext1.localRevocableMemoryContext();
        LocalMemoryContext revocableMemory3 = operatorContext3.localRevocableMemoryContext();
        LocalMemoryContext revocableMemory4 = operatorContext4.localRevocableMemoryContext();
        LocalMemoryContext revocableMemory5 = operatorContext5.localRevocableMemoryContext();
        revocableMemory1.setBytes(3);
        revocableMemory3.setBytes(6);
        assertEquals(1, memoryPool.getFreeBytes());
        scheduler.awaitAsynchronousCallbacksRun();
        // we are still good - no revoking needed
        assertMemoryRevokingNotRequested();
        revocableMemory4.setBytes(7);
        assertEquals(-6, memoryPool.getFreeBytes());
        scheduler.awaitAsynchronousCallbacksRun();
        // we need to revoke 3 and 6
        assertMemoryRevokingRequestedFor(operatorContext1, operatorContext3);
        // lets revoke some bytes
        revocableMemory1.setBytes(0);
        operatorContext1.resetMemoryRevokingRequested();
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingRequestedFor(operatorContext3);
        assertEquals(-3, memoryPool.getFreeBytes());
        // and allocate some more
        revocableMemory5.setBytes(3);
        assertEquals(-6, memoryPool.getFreeBytes());
        scheduler.awaitAsynchronousCallbacksRun();
        // we are still good with just OC3 in process of revoking
        assertMemoryRevokingRequestedFor(operatorContext3);
        // and allocate some more
        revocableMemory5.setBytes(4);
        assertEquals(-7, memoryPool.getFreeBytes());
        scheduler.awaitAsynchronousCallbacksRun();
        // now we have to trigger revoking for OC4
        assertMemoryRevokingRequestedFor(operatorContext3, operatorContext4);
    } finally {
        scheduler.stop();
    }
}

Example 58 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class TestMemoryRevokingScheduler method testQueryMemoryRevoking.

@Test
public void testQueryMemoryRevoking() throws Exception {
    // The various tasks created here use a small amount of system memory independent of what's set explicitly
    // in this test. Triggering spilling based on differences of thousands of bytes rather than hundreds
    // makes the test resilient to any noise that creates.
    // There can still be a race condition where some of these allocations are made when the total memory is above
    // the spill threshold, but in revokeMemory() some memory is reduced between when we get the total memory usage
    // and when we get the task memory usage.  This can cause some extra spilling.
    // To prevent flakiness in the test, we reset revoke memory requested for all operators, even if only one spilled.
    QueryId queryId = new QueryId("query");
    // use a larger memory pool so that we don't trigger spilling due to filling the memory pool
    MemoryPool queryLimitMemoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(100, GIGABYTE));
    SqlTask sqlTask1 = newSqlTask(queryId, queryLimitMemoryPool);
    TestOperatorContext operatorContext11 = createTestingOperatorContexts(sqlTask1, "operator11");
    TestOperatorContext operatorContext12 = createTestingOperatorContexts(sqlTask1, "operator12");
    SqlTask sqlTask2 = newSqlTask(queryId, queryLimitMemoryPool);
    TestOperatorContext operatorContext2 = createTestingOperatorContexts(sqlTask2, "operator2");
    allOperatorContexts = ImmutableSet.of(operatorContext11, operatorContext12, operatorContext2);
    List<SqlTask> tasks = ImmutableList.of(sqlTask1, sqlTask2);
    MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(singletonList(queryLimitMemoryPool), () -> tasks, queryContexts::get, 1.0, 1.0, ORDER_BY_REVOCABLE_BYTES, true);
    try {
        scheduler.start();
        assertMemoryRevokingNotRequested();
        operatorContext11.localRevocableMemoryContext().setBytes(150_000);
        operatorContext2.localRevocableMemoryContext().setBytes(100_000);
        // at this point, Task1 = 150k total bytes, Task2 = 100k total bytes
        // this ensures that we are waiting for the memory revocation listener and not using polling-based revoking
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingNotRequested();
        operatorContext12.localRevocableMemoryContext().setBytes(300_000);
        // at this point, Task1 =  450k total bytes, Task2 = 100k total bytes
        scheduler.awaitAsynchronousCallbacksRun();
        // only operator11 should revoke since we need to revoke only 50k bytes
        // limit - (task1 + task2) => 500k - (450k + 100k) = 50k byte to revoke
        assertMemoryRevokingRequestedFor(operatorContext11);
        // revoke all bytes in operator11
        operatorContext11.localRevocableMemoryContext().setBytes(0);
        // at this point, Task1 = 300k total bytes, Task2 = 100k total bytes
        scheduler.awaitAsynchronousCallbacksRun();
        operatorContext11.resetMemoryRevokingRequested();
        operatorContext12.resetMemoryRevokingRequested();
        operatorContext2.resetMemoryRevokingRequested();
        assertMemoryRevokingNotRequested();
        operatorContext11.localRevocableMemoryContext().setBytes(20_000);
        // at this point, Task1 = 320,000 total bytes (oc11 - 20k, oc12 - 300k), Task2 = 100k total bytes
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingNotRequested();
        operatorContext2.localSystemMemoryContext().setBytes(150_000);
        // at this point, Task1 = 320K total bytes, Task2 = 250K total bytes
        // both operator11 and operator 12 are revoking since we revoke in order of operator creation within the task until we are below the memory revoking threshold
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingRequestedFor(operatorContext11, operatorContext12);
        operatorContext11.localRevocableMemoryContext().setBytes(0);
        operatorContext12.localRevocableMemoryContext().setBytes(0);
        scheduler.awaitAsynchronousCallbacksRun();
        operatorContext11.resetMemoryRevokingRequested();
        operatorContext12.resetMemoryRevokingRequested();
        operatorContext2.resetMemoryRevokingRequested();
        assertMemoryRevokingNotRequested();
        operatorContext11.localRevocableMemoryContext().setBytes(50_000);
        operatorContext12.localRevocableMemoryContext().setBytes(50_000);
        operatorContext2.localSystemMemoryContext().setBytes(150_000);
        operatorContext2.localRevocableMemoryContext().setBytes(150_000);
        scheduler.awaitAsynchronousCallbacksRun();
        // no need to revoke
        assertMemoryRevokingNotRequested();
        // at this point, Task1 = 75k total bytes, Task2 = 300k total bytes (150k revocable, 150k system)
        operatorContext12.localUserMemoryContext().setBytes(300_000);
        // at this point, Task1 = 400K total bytes (100k revocable, 300k user), Task2 = 300k total bytes (150k revocable, 150k system)
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingRequestedFor(operatorContext2, operatorContext11);
    } finally {
        scheduler.stop();
    }
}

Example 59 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class TestQueryIdGenerator method testCreateNextQueryId.

@Test
public void testCreateNextQueryId() {
    TestIdGenerator idGenerator = new TestIdGenerator();
    long millis = new DateTime(2001, 7, 14, 1, 2, 3, 4, DateTimeZone.UTC).getMillis();
    idGenerator.setNow(millis);
    // generate ids to 99,999
    for (int i = 0; i < 100_000; i++) {
        assertEquals(idGenerator.createNextQueryId(), new QueryId(String.format("20010714_010203_%05d_%s", i, idGenerator.getCoordinatorId())));
    }
    // next id will cause counter to roll, but we need to add a second to the time or code will block for ever
    millis += 1000;
    idGenerator.setNow(millis);
    for (int i = 0; i < 100_000; i++) {
        assertEquals(idGenerator.createNextQueryId(), new QueryId(String.format("20010714_010204_%05d_%s", i, idGenerator.getCoordinatorId())));
    }
    // more forward one more second and generate 100 ids
    millis += 1000;
    idGenerator.setNow(millis);
    for (int i = 0; i < 100; i++) {
        assertEquals(idGenerator.createNextQueryId(), new QueryId(String.format("20010714_010205_%05d_%s", i, idGenerator.getCoordinatorId())));
    }
    // now we move to the start of the next day, and the counter should reset
    millis = new DateTime(2001, 7, 15, 0, 0, 0, 0, DateTimeZone.UTC).getMillis();
    idGenerator.setNow(millis);
    for (int i = 0; i < 100_000; i++) {
        assertEquals(idGenerator.createNextQueryId(), new QueryId(String.format("20010715_000000_%05d_%s", i, idGenerator.getCoordinatorId())));
    }
}

Example 60 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class TestMemoryRevokingScheduler method testRevokesPoolWhenFullBeforeQueryLimit.

@Test
public void testRevokesPoolWhenFullBeforeQueryLimit() throws Exception {
    QueryContext q1 = getOrCreateQueryContext(new QueryId("q1"), memoryPool);
    QueryContext q2 = getOrCreateQueryContext(new QueryId("q2"), memoryPool);
    SqlTask sqlTask1 = newSqlTask(q1.getQueryId(), memoryPool);
    SqlTask sqlTask2 = newSqlTask(q2.getQueryId(), memoryPool);
    TaskContext taskContext1 = getOrCreateTaskContext(sqlTask1);
    PipelineContext pipelineContext11 = taskContext1.addPipelineContext(0, false, false, false);
    DriverContext driverContext111 = pipelineContext11.addDriverContext();
    OperatorContext operatorContext1 = driverContext111.addOperatorContext(1, new PlanNodeId("na"), "na");
    OperatorContext operatorContext2 = driverContext111.addOperatorContext(2, new PlanNodeId("na"), "na");
    DriverContext driverContext112 = pipelineContext11.addDriverContext();
    OperatorContext operatorContext3 = driverContext112.addOperatorContext(3, new PlanNodeId("na"), "na");
    TaskContext taskContext2 = getOrCreateTaskContext(sqlTask2);
    PipelineContext pipelineContext21 = taskContext2.addPipelineContext(1, false, false, false);
    DriverContext driverContext211 = pipelineContext21.addDriverContext();
    OperatorContext operatorContext4 = driverContext211.addOperatorContext(4, new PlanNodeId("na"), "na");
    List<SqlTask> tasks = ImmutableList.of(sqlTask1, sqlTask2);
    MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(singletonList(memoryPool), () -> tasks, queryContexts::get, 1.0, 1.0, ORDER_BY_CREATE_TIME, true);
    try {
        scheduler.start();
        allOperatorContexts = ImmutableSet.of(operatorContext1, operatorContext2, operatorContext3, operatorContext4);
        assertMemoryRevokingNotRequested();
        assertEquals(10, memoryPool.getFreeBytes());
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingNotRequested();
        LocalMemoryContext revocableMemory1 = operatorContext1.localRevocableMemoryContext();
        LocalMemoryContext revocableMemory3 = operatorContext3.localRevocableMemoryContext();
        LocalMemoryContext revocableMemory4 = operatorContext4.localRevocableMemoryContext();
        revocableMemory1.setBytes(3);
        revocableMemory3.setBytes(6);
        assertEquals(1, memoryPool.getFreeBytes());
        scheduler.awaitAsynchronousCallbacksRun();
        // we are still good - no revoking needed
        assertMemoryRevokingNotRequested();
        revocableMemory4.setBytes(7);
        assertEquals(-6, memoryPool.getFreeBytes());
        scheduler.awaitAsynchronousCallbacksRun();
        // we need to revoke 3 and 6
        assertMemoryRevokingRequestedFor(operatorContext1, operatorContext3);
    } finally {
        scheduler.stop();
    }
}