Examples with StageId io.trino.execution.StageId used on opensource projects

Example 21 with StageId

use of io.trino.execution.StageId in project trino by trinodb.

the class TestTaskExecutor method testQuantaFairness.

@Test(invocationCount = 100)
public void testQuantaFairness() {
    TestingTicker ticker = new TestingTicker();
    TaskExecutor taskExecutor = new TaskExecutor(1, 2, 3, 4, ticker);
    taskExecutor.start();
    ticker.increment(20, MILLISECONDS);
    try {
        TaskHandle shortQuantaTaskHandle = taskExecutor.addTask(new TaskId(new StageId("short_quanta", 0), 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty());
        TaskHandle longQuantaTaskHandle = taskExecutor.addTask(new TaskId(new StageId("long_quanta", 0), 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty());
        Phaser endQuantaPhaser = new Phaser();
        TestingJob shortQuantaDriver = new TestingJob(ticker, new Phaser(), new Phaser(), endQuantaPhaser, 10, 10);
        TestingJob longQuantaDriver = new TestingJob(ticker, new Phaser(), new Phaser(), endQuantaPhaser, 10, 20);
        taskExecutor.enqueueSplits(shortQuantaTaskHandle, true, ImmutableList.of(shortQuantaDriver));
        taskExecutor.enqueueSplits(longQuantaTaskHandle, true, ImmutableList.of(longQuantaDriver));
        for (int i = 0; i < 11; i++) {
            endQuantaPhaser.arriveAndAwaitAdvance();
        }
        assertTrue(shortQuantaDriver.getCompletedPhases() >= 7 && shortQuantaDriver.getCompletedPhases() <= 8);
        assertTrue(longQuantaDriver.getCompletedPhases() >= 3 && longQuantaDriver.getCompletedPhases() <= 4);
        endQuantaPhaser.arriveAndDeregister();
    } finally {
        taskExecutor.stop();
    }
}

Example 22 with StageId

use of io.trino.execution.StageId in project trino by trinodb.

the class TestTaskExecutor method testMinDriversPerTaskWhenTargetConcurrencyIncreases.

@Test
public void testMinDriversPerTaskWhenTargetConcurrencyIncreases() {
    MultilevelSplitQueue splitQueue = new MultilevelSplitQueue(2);
    TestingTicker ticker = new TestingTicker();
    // create a task executor with min/max drivers per task to be 2
    TaskExecutor taskExecutor = new TaskExecutor(4, 1, 2, 2, splitQueue, ticker);
    taskExecutor.start();
    try {
        TaskHandle testTaskHandle = taskExecutor.addTask(new TaskId(new StageId(new QueryId("test"), 0), 0, 0), // make sure buffer is underutilized
        () -> 0, 1, new Duration(1, MILLISECONDS), OptionalInt.of(2));
        // create 3 splits
        int batchCount = 3;
        TestingJob[] splits = new TestingJob[3];
        Phaser[] phasers = new Phaser[batchCount];
        for (int batch = 0; batch < batchCount; batch++) {
            phasers[batch] = new Phaser();
            phasers[batch].register();
            TestingJob split = new TestingJob(ticker, new Phaser(), new Phaser(), phasers[batch], 1, 0);
            splits[batch] = split;
        }
        taskExecutor.enqueueSplits(testTaskHandle, false, ImmutableList.copyOf(splits));
        // wait until first split starts
        waitUntilSplitsStart(ImmutableList.of(splits[0]));
        // remaining splits shouldn't start because initial split concurrency is 1
        assertSplitStates(0, splits);
        // complete first split (SplitConcurrencyController for TaskHandle should increase concurrency since buffer is underutilized)
        phasers[0].arriveAndDeregister();
        // 2 remaining splits should be started
        waitUntilSplitsStart(ImmutableList.of(splits[1], splits[2]));
    } finally {
        taskExecutor.stop();
    }
}

Example 23 with StageId

use of io.trino.execution.StageId in project trino by trinodb.

the class TestTaskExecutor method testLevelMultipliers.

@Test(invocationCount = 100)
public void testLevelMultipliers() throws Exception {
    TestingTicker ticker = new TestingTicker();
    TaskExecutor taskExecutor = new TaskExecutor(6, 3, 3, 4, new MultilevelSplitQueue(2), ticker);
    taskExecutor.start();
    ticker.increment(20, MILLISECONDS);
    try {
        for (int i = 0; i < (LEVEL_THRESHOLD_SECONDS.length - 1); i++) {
            TaskHandle[] taskHandles = { taskExecutor.addTask(new TaskId(new StageId("test1", 0), 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()), taskExecutor.addTask(new TaskId(new StageId("test2", 0), 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()), taskExecutor.addTask(new TaskId(new StageId("test3", 0), 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()) };
            // move task 0 to next level
            TestingJob task0Job = new TestingJob(ticker, new Phaser(), new Phaser(), new Phaser(), 1, LEVEL_THRESHOLD_SECONDS[i + 1] * 1000);
            taskExecutor.enqueueSplits(taskHandles[0], true, ImmutableList.of(task0Job));
            // move tasks 1 and 2 to this level
            TestingJob task1Job = new TestingJob(ticker, new Phaser(), new Phaser(), new Phaser(), 1, LEVEL_THRESHOLD_SECONDS[i] * 1000);
            taskExecutor.enqueueSplits(taskHandles[1], true, ImmutableList.of(task1Job));
            TestingJob task2Job = new TestingJob(ticker, new Phaser(), new Phaser(), new Phaser(), 1, LEVEL_THRESHOLD_SECONDS[i] * 1000);
            taskExecutor.enqueueSplits(taskHandles[2], true, ImmutableList.of(task2Job));
            task0Job.getCompletedFuture().get();
            task1Job.getCompletedFuture().get();
            task2Job.getCompletedFuture().get();
            // then, start new drivers for all tasks
            // 6 taskExecutor threads + test thread
            Phaser globalPhaser = new Phaser(7);
            int phasesForNextLevel = LEVEL_THRESHOLD_SECONDS[i + 1] - LEVEL_THRESHOLD_SECONDS[i];
            TestingJob[] drivers = new TestingJob[6];
            for (int j = 0; j < 6; j++) {
                drivers[j] = new TestingJob(ticker, globalPhaser, new Phaser(), new Phaser(), phasesForNextLevel, 1000);
            }
            taskExecutor.enqueueSplits(taskHandles[0], true, ImmutableList.of(drivers[0], drivers[1]));
            taskExecutor.enqueueSplits(taskHandles[1], true, ImmutableList.of(drivers[2], drivers[3]));
            taskExecutor.enqueueSplits(taskHandles[2], true, ImmutableList.of(drivers[4], drivers[5]));
            // run all three drivers
            int lowerLevelStart = drivers[2].getCompletedPhases() + drivers[3].getCompletedPhases() + drivers[4].getCompletedPhases() + drivers[5].getCompletedPhases();
            int higherLevelStart = drivers[0].getCompletedPhases() + drivers[1].getCompletedPhases();
            while (Arrays.stream(drivers).noneMatch(TestingJob::isFinished)) {
                globalPhaser.arriveAndAwaitAdvance();
                int lowerLevelEnd = drivers[2].getCompletedPhases() + drivers[3].getCompletedPhases() + drivers[4].getCompletedPhases() + drivers[5].getCompletedPhases();
                int lowerLevelTime = lowerLevelEnd - lowerLevelStart;
                int higherLevelEnd = drivers[0].getCompletedPhases() + drivers[1].getCompletedPhases();
                int higherLevelTime = higherLevelEnd - higherLevelStart;
                if (higherLevelTime > 20) {
                    assertGreaterThan(lowerLevelTime, (higherLevelTime * 2) - 10);
                    assertLessThan(higherLevelTime, (lowerLevelTime * 2) + 10);
                }
            }
            globalPhaser.arriveAndDeregister();
            taskExecutor.removeTask(taskHandles[0]);
            taskExecutor.removeTask(taskHandles[1]);
            taskExecutor.removeTask(taskHandles[2]);
        }
    } finally {
        taskExecutor.stop();
    }
}

Example 24 with StageId

use of io.trino.execution.StageId in project trino by trinodb.

the class TestTaskExecutor method testMinMaxDriversPerTask.

@Test(timeOut = 30_000)
public void testMinMaxDriversPerTask() {
    int maxDriversPerTask = 2;
    MultilevelSplitQueue splitQueue = new MultilevelSplitQueue(2);
    TestingTicker ticker = new TestingTicker();
    TaskExecutor taskExecutor = new TaskExecutor(4, 16, 1, maxDriversPerTask, splitQueue, ticker);
    taskExecutor.start();
    try {
        TaskHandle testTaskHandle = taskExecutor.addTask(new TaskId(new StageId("test", 0), 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty());
        // enqueue all batches of splits
        int batchCount = 4;
        TestingJob[] splits = new TestingJob[8];
        Phaser[] phasers = new Phaser[batchCount];
        for (int batch = 0; batch < batchCount; batch++) {
            phasers[batch] = new Phaser();
            phasers[batch].register();
            TestingJob split1 = new TestingJob(ticker, new Phaser(), new Phaser(), phasers[batch], 1, 0);
            TestingJob split2 = new TestingJob(ticker, new Phaser(), new Phaser(), phasers[batch], 1, 0);
            splits[2 * batch] = split1;
            splits[2 * batch + 1] = split2;
            taskExecutor.enqueueSplits(testTaskHandle, false, ImmutableList.of(split1, split2));
        }
        // assert that the splits are processed in batches as expected
        for (int batch = 0; batch < batchCount; batch++) {
            // wait until the current batch starts
            waitUntilSplitsStart(ImmutableList.of(splits[2 * batch], splits[2 * batch + 1]));
            // assert that only the splits including and up to the current batch are running and the rest haven't started yet
            assertSplitStates(2 * batch + 1, splits);
            // complete the current batch
            phasers[batch].arriveAndDeregister();
        }
    } finally {
        taskExecutor.stop();
    }
}

Example 25 with StageId

use of io.trino.execution.StageId in project trino by trinodb.

the class TestTaskExecutor method testLevelMovement.

@Test(invocationCount = 100)
public void testLevelMovement() {
    TestingTicker ticker = new TestingTicker();
    TaskExecutor taskExecutor = new TaskExecutor(2, 2, 3, 4, ticker);
    taskExecutor.start();
    ticker.increment(20, MILLISECONDS);
    try {
        TaskHandle testTaskHandle = taskExecutor.addTask(new TaskId(new StageId("test", 0), 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty());
        Phaser globalPhaser = new Phaser();
        // 2 taskExecutor threads + test thread
        globalPhaser.bulkRegister(3);
        int quantaTimeMills = 500;
        int phasesPerSecond = 1000 / quantaTimeMills;
        int totalPhases = LEVEL_THRESHOLD_SECONDS[LEVEL_THRESHOLD_SECONDS.length - 1] * phasesPerSecond;
        TestingJob driver1 = new TestingJob(ticker, globalPhaser, new Phaser(), new Phaser(), totalPhases, quantaTimeMills);
        TestingJob driver2 = new TestingJob(ticker, globalPhaser, new Phaser(), new Phaser(), totalPhases, quantaTimeMills);
        taskExecutor.enqueueSplits(testTaskHandle, true, ImmutableList.of(driver1, driver2));
        int completedPhases = 0;
        for (int i = 0; i < (LEVEL_THRESHOLD_SECONDS.length - 1); i++) {
            for (; (completedPhases / phasesPerSecond) < LEVEL_THRESHOLD_SECONDS[i + 1]; completedPhases++) {
                globalPhaser.arriveAndAwaitAdvance();
            }
            assertEquals(testTaskHandle.getPriority().getLevel(), i + 1);
        }
        globalPhaser.arriveAndDeregister();
    } finally {
        taskExecutor.stop();
    }
}