Examples with QueryId com.facebook.presto.spi.QueryId used on opensource projects

Example 76 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class TestMemoryRevokingScheduler method testTaskRevokingOrderForCreateTime.

/**
 * Ensures that when revoking is requested, the first task to start revoking is based on the {@link FeaturesConfig.TaskSpillingStrategy}
 */
@Test
public void testTaskRevokingOrderForCreateTime() throws Exception {
    SqlTask sqlTask1 = newSqlTask(new QueryId("query"), memoryPool);
    TestOperatorContext operatorContext1 = createTestingOperatorContexts(sqlTask1, "operator1");
    SqlTask sqlTask2 = newSqlTask(new QueryId("query"), memoryPool);
    TestOperatorContext operatorContext2 = createTestingOperatorContexts(sqlTask2, "operator2");
    allOperatorContexts = ImmutableSet.of(operatorContext1, operatorContext2);
    List<SqlTask> tasks = ImmutableList.of(sqlTask1, sqlTask2);
    MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(singletonList(memoryPool), () -> tasks, queryContexts::get, 1.0, 1.0, ORDER_BY_CREATE_TIME, false);
    try {
        // no periodic check initiated
        scheduler.start();
        assertMemoryRevokingNotRequested();
        operatorContext1.localRevocableMemoryContext().setBytes(11);
        operatorContext2.localRevocableMemoryContext().setBytes(12);
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingRequestedFor(operatorContext1, operatorContext2);
        // operator1 should revoke first as it belongs to a task that was created earlier
        assertEquals(TestOperatorContext.firstOperator, "operator1");
    } finally {
        scheduler.stop();
    }
}

Example 77 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class TestMemoryRevokingScheduler method testCountAlreadyRevokedMemoryWithinAPool.

@Test
public void testCountAlreadyRevokedMemoryWithinAPool() throws Exception {
    // Given
    MemoryPool anotherMemoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(10, BYTE));
    SqlTask sqlTask1 = newSqlTask(new QueryId("q1"), anotherMemoryPool);
    OperatorContext operatorContext1 = createContexts(sqlTask1);
    SqlTask sqlTask2 = newSqlTask(new QueryId("q2"), memoryPool);
    OperatorContext operatorContext2 = createContexts(sqlTask2);
    List<SqlTask> tasks = ImmutableList.of(sqlTask1, sqlTask2);
    MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(asList(memoryPool, anotherMemoryPool), () -> tasks, queryContexts::get, 1.0, 1.0, ORDER_BY_CREATE_TIME, false);
    try {
        scheduler.start();
        allOperatorContexts = ImmutableSet.of(operatorContext1, operatorContext2);
        /*
             * sqlTask1 fills its pool
             */
        operatorContext1.localRevocableMemoryContext().setBytes(12);
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingRequestedFor(operatorContext1);
        /*
             * When sqlTask2 fills its pool
             */
        operatorContext2.localRevocableMemoryContext().setBytes(12);
        scheduler.awaitAsynchronousCallbacksRun();
        /*
             * Then sqlTask2 should be asked to revoke its memory too
             */
        assertMemoryRevokingRequestedFor(operatorContext1, operatorContext2);
    } finally {
        scheduler.stop();
    }
}

Example 78 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class TestMemoryRevokingScheduler method testTaskThresholdRevokingSchedulerImmediate.

@Test
public void testTaskThresholdRevokingSchedulerImmediate() throws Exception {
    SqlTask sqlTask1 = newSqlTask(new QueryId("query"), memoryPool);
    TestOperatorContext operatorContext11 = createTestingOperatorContexts(sqlTask1, "operator11");
    TestOperatorContext operatorContext12 = createTestingOperatorContexts(sqlTask1, "operator12");
    SqlTask sqlTask2 = newSqlTask(new QueryId("query"), memoryPool);
    TestOperatorContext operatorContext2 = createTestingOperatorContexts(sqlTask2, "operator2");
    allOperatorContexts = ImmutableSet.of(operatorContext11, operatorContext12, operatorContext2);
    List<SqlTask> tasks = ImmutableList.of(sqlTask1, sqlTask2);
    ImmutableMap<TaskId, SqlTask> taskMap = ImmutableMap.of(sqlTask1.getTaskId(), sqlTask1, sqlTask2.getTaskId(), sqlTask2);
    TaskThresholdMemoryRevokingScheduler scheduler = new TaskThresholdMemoryRevokingScheduler(singletonList(memoryPool), () -> tasks, taskMap::get, singleThreadedScheduledExecutor, 5L);
    // no periodic check initiated
    scheduler.registerPoolListeners();
    assertMemoryRevokingNotRequested();
    operatorContext11.localRevocableMemoryContext().setBytes(3);
    operatorContext2.localRevocableMemoryContext().setBytes(2);
    // at this point, Task1 = 3 total bytes, Task2 = 2 total bytes
    // this ensures that we are waiting for the memory revocation listener and not using polling-based revoking
    awaitTaskThresholdAsynchronousCallbacksRun();
    assertMemoryRevokingNotRequested();
    operatorContext12.localRevocableMemoryContext().setBytes(3);
    // at this point, Task1 = 6 total bytes, Task2 = 2 total bytes
    awaitTaskThresholdAsynchronousCallbacksRun();
    // only operator11 should revoke since we need to revoke only 1 byte
    // threshold - (operator11 + operator12) => 5 - (3 + 3) = 1 bytes to revoke
    assertMemoryRevokingRequestedFor(operatorContext11);
    // revoke 2 bytes in operator11
    operatorContext11.localRevocableMemoryContext().setBytes(1);
    // at this point, Task1 = 3 total bytes, Task2 = 2 total bytes
    operatorContext11.resetMemoryRevokingRequested();
    awaitTaskThresholdAsynchronousCallbacksRun();
    assertMemoryRevokingNotRequested();
    // operator12 fills up
    operatorContext12.localRevocableMemoryContext().setBytes(6);
    // at this point, Task1 = 7 total bytes, Task2 = 2 total bytes
    awaitTaskThresholdAsynchronousCallbacksRun();
    // both operator11 and operator 12 are revoking since we revoke in order of operator creation within the task until we are below the memory revoking threshold
    assertMemoryRevokingRequestedFor(operatorContext11, operatorContext12);
    operatorContext11.localRevocableMemoryContext().setBytes(2);
    operatorContext11.resetMemoryRevokingRequested();
    operatorContext12.localRevocableMemoryContext().setBytes(2);
    operatorContext12.resetMemoryRevokingRequested();
    // at this point, Task1 = 4 total bytes, Task2 = 2 total bytes
    awaitTaskThresholdAsynchronousCallbacksRun();
    // no need to revoke
    assertMemoryRevokingNotRequested();
    operatorContext2.localRevocableMemoryContext().setBytes(6);
    // at this point, Task1 = 4 total bytes, Task2 = 6 total bytes, operators in Task2 must be revoked
    awaitTaskThresholdAsynchronousCallbacksRun();
    assertMemoryRevokingRequestedFor(operatorContext2);
}

Example 79 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class TestMemoryRevokingScheduler method testTaskThresholdRevokingScheduler.

@Test
public void testTaskThresholdRevokingScheduler() throws Exception {
    SqlTask sqlTask1 = newSqlTask(new QueryId("query"), memoryPool);
    TestOperatorContext operatorContext11 = createTestingOperatorContexts(sqlTask1, "operator11");
    TestOperatorContext operatorContext12 = createTestingOperatorContexts(sqlTask1, "operator12");
    SqlTask sqlTask2 = newSqlTask(new QueryId("query"), memoryPool);
    TestOperatorContext operatorContext2 = createTestingOperatorContexts(sqlTask2, "operator2");
    allOperatorContexts = ImmutableSet.of(operatorContext11, operatorContext12, operatorContext2);
    List<SqlTask> tasks = ImmutableList.of(sqlTask1, sqlTask2);
    ImmutableMap<TaskId, SqlTask> taskMap = ImmutableMap.of(sqlTask1.getTaskId(), sqlTask1, sqlTask2.getTaskId(), sqlTask2);
    TaskThresholdMemoryRevokingScheduler scheduler = new TaskThresholdMemoryRevokingScheduler(singletonList(memoryPool), () -> tasks, taskMap::get, singleThreadedScheduledExecutor, 5L);
    assertMemoryRevokingNotRequested();
    operatorContext11.localRevocableMemoryContext().setBytes(3);
    operatorContext2.localRevocableMemoryContext().setBytes(2);
    // at this point, Task1 = 3 total bytes, Task2 = 2 total bytes
    requestMemoryRevoking(scheduler);
    assertMemoryRevokingNotRequested();
    operatorContext12.localRevocableMemoryContext().setBytes(3);
    // at this point, Task1 = 6 total bytes, Task2 = 2 total bytes
    requestMemoryRevoking(scheduler);
    // only operator11 should revoke since we need to revoke only 1 byte
    // threshold - (operator11 + operator12) => 5 - (3 + 3) = 1 bytes to revoke
    assertMemoryRevokingRequestedFor(operatorContext11);
    // revoke 2 bytes in operator11
    operatorContext11.localRevocableMemoryContext().setBytes(1);
    // at this point, Task1 = 3 total bytes, Task2 = 2 total bytes
    operatorContext11.resetMemoryRevokingRequested();
    requestMemoryRevoking(scheduler);
    assertMemoryRevokingNotRequested();
    // operator12 fills up
    operatorContext12.localRevocableMemoryContext().setBytes(6);
    // at this point, Task1 = 7 total bytes, Task2 = 2 total bytes
    requestMemoryRevoking(scheduler);
    // both operator11 and operator 12 are revoking since we revoke in order of operator creation within the task until we are below the memory revoking threshold
    assertMemoryRevokingRequestedFor(operatorContext11, operatorContext12);
    operatorContext11.localRevocableMemoryContext().setBytes(2);
    operatorContext11.resetMemoryRevokingRequested();
    operatorContext12.localRevocableMemoryContext().setBytes(2);
    operatorContext12.resetMemoryRevokingRequested();
    // at this point, Task1 = 4 total bytes, Task2 = 2 total bytes
    requestMemoryRevoking(scheduler);
    // no need to revoke
    assertMemoryRevokingNotRequested();
    operatorContext2.localRevocableMemoryContext().setBytes(6);
    // at this point, Task1 = 4 total bytes, Task2 = 6 total bytes, operators in Task2 must be revoked
    requestMemoryRevoking(scheduler);
    assertMemoryRevokingRequestedFor(operatorContext2);
}

Example 80 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class TestMemoryRevokingScheduler method testTaskRevokingOrderForRevocableBytes.

@Test
public void testTaskRevokingOrderForRevocableBytes() throws Exception {
    SqlTask sqlTask1 = newSqlTask(new QueryId("query"), memoryPool);
    TestOperatorContext operatorContext1 = createTestingOperatorContexts(sqlTask1, "operator1");
    SqlTask sqlTask2 = newSqlTask(new QueryId("query"), memoryPool);
    TestOperatorContext operatorContext2 = createTestingOperatorContexts(sqlTask2, "operator2");
    allOperatorContexts = ImmutableSet.of(operatorContext1, operatorContext2);
    List<SqlTask> tasks = ImmutableList.of(sqlTask1, sqlTask2);
    MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(singletonList(memoryPool), () -> tasks, queryContexts::get, 1.0, 1.0, ORDER_BY_REVOCABLE_BYTES, false);
    try {
        scheduler.start();
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingNotRequested();
        operatorContext1.localRevocableMemoryContext().setBytes(11);
        operatorContext2.localRevocableMemoryContext().setBytes(12);
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingRequestedFor(operatorContext1, operatorContext2);
        // operator2 should revoke first since it (and it's encompassing task) has allocated more bytes
        assertEquals(TestOperatorContext.firstOperator, "operator2");
    } finally {
        scheduler.stop();
    }
}