Examples with TestingTicker com.facebook.airlift.testing.TestingTicker used on opensource projects

Example 11 with TestingTicker

use of com.facebook.airlift.testing.TestingTicker in project presto by prestodb.

the class TestPageProcessor method testExpressionProfiler.

@Test
public void testExpressionProfiler() {
    MetadataManager metadata = createTestMetadataManager();
    CallExpression add10Expression = call(ADD.name(), metadata.getFunctionAndTypeManager().resolveOperator(ADD, fromTypes(BIGINT, BIGINT)), BIGINT, field(0, BIGINT), constant(10L, BIGINT));
    TestingTicker testingTicker = new TestingTicker();
    PageFunctionCompiler functionCompiler = new PageFunctionCompiler(metadata, 0);
    Supplier<PageProjection> projectionSupplier = functionCompiler.compileProjection(SESSION.getSqlFunctionProperties(), add10Expression, Optional.empty());
    PageProjection projection = projectionSupplier.get();
    Page page = new Page(createLongSequenceBlock(1, 11));
    ExpressionProfiler profiler = new ExpressionProfiler(testingTicker, SPLIT_RUN_QUANTA);
    for (int i = 0; i < 100; i++) {
        profiler.start();
        Work<List<Block>> work = projection.project(SESSION.getSqlFunctionProperties(), new DriverYieldSignal(), page, SelectedPositions.positionsRange(0, page.getPositionCount()));
        if (i < 10) {
            // increment the ticker with a large value to mark the expression as expensive
            testingTicker.increment(10, SECONDS);
            profiler.stop(page.getPositionCount());
            assertTrue(profiler.isExpressionExpensive());
        } else {
            testingTicker.increment(0, NANOSECONDS);
            profiler.stop(page.getPositionCount());
            assertFalse(profiler.isExpressionExpensive());
        }
        work.process();
    }
}

Example 12 with TestingTicker

use of com.facebook.airlift.testing.TestingTicker in project presto by prestodb.

the class TestPageProcessor method testIncreasingBatchSize.

@Test
public void testIncreasingBatchSize() {
    int rows = 1024;
    // We deliberately do not set the ticker, so that the expression is always cheap and the batch size gets doubled until other limits are hit
    TestingTicker testingTicker = new TestingTicker();
    ExpressionProfiler profiler = new ExpressionProfiler(testingTicker, SPLIT_RUN_QUANTA);
    PageProcessor pageProcessor = new PageProcessor(Optional.empty(), ImmutableList.of(createInputPageProjectionWithOutputs(0, BIGINT, 0)), OptionalInt.of(1), profiler);
    Slice[] slices = new Slice[rows];
    Arrays.fill(slices, Slices.allocate(rows));
    Page inputPage = new Page(createSlicesBlock(slices));
    Iterator<Optional<Page>> output = processAndAssertRetainedPageSize(pageProcessor, inputPage);
    long previousPositionCount = 1;
    long totalPositionCount = 0;
    while (totalPositionCount < rows) {
        Optional<Page> page = output.next();
        assertTrue(page.isPresent());
        long positionCount = page.get().getPositionCount();
        totalPositionCount += positionCount;
        // skip the first read && skip the last read, which can be a partial page
        if (positionCount > 1 && totalPositionCount != rows) {
            assertEquals(positionCount, previousPositionCount * 2);
        }
        previousPositionCount = positionCount;
    }
}

Example 13 with TestingTicker

use of com.facebook.airlift.testing.TestingTicker in project presto by prestodb.

the class TestPageProcessor method testDecreasingBatchSize.

@Test
public void testDecreasingBatchSize() {
    int rows = 1024;
    // We set the expensive expression threshold to 0, so the expression is always considered expensive and the batch size gets halved until it becomes 1
    TestingTicker testingTicker = new TestingTicker();
    ExpressionProfiler profiler = new ExpressionProfiler(testingTicker, new Duration(0, MILLISECONDS));
    PageProcessor pageProcessor = new PageProcessor(Optional.empty(), ImmutableList.of(createInputPageProjectionWithOutputs(0, BIGINT, 0)), OptionalInt.of(512), profiler);
    Slice[] slices = new Slice[rows];
    Arrays.fill(slices, Slices.allocate(rows));
    Page inputPage = new Page(createSlicesBlock(slices));
    Iterator<Optional<Page>> output = processAndAssertRetainedPageSize(pageProcessor, inputPage);
    long previousPositionCount = 1;
    long totalPositionCount = 0;
    while (totalPositionCount < rows) {
        Optional<Page> page = output.next();
        assertTrue(page.isPresent());
        long positionCount = page.get().getPositionCount();
        totalPositionCount += positionCount;
        // the batch size doesn't get smaller than 1
        if (positionCount > 1 && previousPositionCount != 1) {
            assertEquals(positionCount, previousPositionCount / 2);
        }
        previousPositionCount = positionCount;
    }
}

Example 14 with TestingTicker

use of com.facebook.airlift.testing.TestingTicker in project presto by prestodb.

the class TestTaskExecutor method testTasksComplete.

@Test(invocationCount = 100)
public void testTasksComplete() throws Exception {
    TestingTicker ticker = new TestingTicker();
    TaskExecutor taskExecutor = new TaskExecutor(4, 8, 3, 4, TASK_FAIR, ticker);
    taskExecutor.start();
    ticker.increment(20, MILLISECONDS);
    try {
        TaskId taskId = new TaskId("test", 0, 0, 0);
        TaskHandle taskHandle = taskExecutor.addTask(taskId, () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty());
        Phaser beginPhase = new Phaser();
        beginPhase.register();
        Phaser verificationComplete = new Phaser();
        verificationComplete.register();
        // add two jobs
        TestingJob driver1 = new TestingJob(ticker, new Phaser(1), beginPhase, verificationComplete, 10, 0);
        ListenableFuture<?> future1 = getOnlyElement(taskExecutor.enqueueSplits(taskHandle, true, ImmutableList.of(driver1)));
        TestingJob driver2 = new TestingJob(ticker, new Phaser(1), beginPhase, verificationComplete, 10, 0);
        ListenableFuture<?> future2 = getOnlyElement(taskExecutor.enqueueSplits(taskHandle, true, ImmutableList.of(driver2)));
        assertEquals(driver1.getCompletedPhases(), 0);
        assertEquals(driver2.getCompletedPhases(), 0);
        // verify worker have arrived but haven't processed yet
        beginPhase.arriveAndAwaitAdvance();
        assertEquals(driver1.getCompletedPhases(), 0);
        assertEquals(driver2.getCompletedPhases(), 0);
        ticker.increment(60, SECONDS);
        assertEquals(taskExecutor.getRunAwaySplitCount(), 0);
        ticker.increment(600, SECONDS);
        assertEquals(taskExecutor.getRunAwaySplitCount(), 2);
        verificationComplete.arriveAndAwaitAdvance();
        // advance one phase and verify
        beginPhase.arriveAndAwaitAdvance();
        assertEquals(driver1.getCompletedPhases(), 1);
        assertEquals(driver2.getCompletedPhases(), 1);
        verificationComplete.arriveAndAwaitAdvance();
        // add one more job
        TestingJob driver3 = new TestingJob(ticker, new Phaser(1), beginPhase, verificationComplete, 10, 0);
        ListenableFuture<?> future3 = getOnlyElement(taskExecutor.enqueueSplits(taskHandle, false, ImmutableList.of(driver3)));
        // advance one phase and verify
        beginPhase.arriveAndAwaitAdvance();
        assertEquals(driver1.getCompletedPhases(), 2);
        assertEquals(driver2.getCompletedPhases(), 2);
        assertEquals(driver3.getCompletedPhases(), 0);
        verificationComplete.arriveAndAwaitAdvance();
        // advance to the end of the first two task and verify
        beginPhase.arriveAndAwaitAdvance();
        for (int i = 0; i < 7; i++) {
            verificationComplete.arriveAndAwaitAdvance();
            beginPhase.arriveAndAwaitAdvance();
            assertEquals(beginPhase.getPhase(), verificationComplete.getPhase() + 1);
        }
        assertEquals(driver1.getCompletedPhases(), 10);
        assertEquals(driver2.getCompletedPhases(), 10);
        assertEquals(driver3.getCompletedPhases(), 8);
        future1.get(1, SECONDS);
        future2.get(1, SECONDS);
        verificationComplete.arriveAndAwaitAdvance();
        // advance two more times and verify
        beginPhase.arriveAndAwaitAdvance();
        verificationComplete.arriveAndAwaitAdvance();
        beginPhase.arriveAndAwaitAdvance();
        assertEquals(driver1.getCompletedPhases(), 10);
        assertEquals(driver2.getCompletedPhases(), 10);
        assertEquals(driver3.getCompletedPhases(), 10);
        future3.get(1, SECONDS);
        verificationComplete.arriveAndAwaitAdvance();
        assertEquals(driver1.getFirstPhase(), 0);
        assertEquals(driver2.getFirstPhase(), 0);
        assertEquals(driver3.getFirstPhase(), 2);
        assertEquals(driver1.getLastPhase(), 10);
        assertEquals(driver2.getLastPhase(), 10);
        assertEquals(driver3.getLastPhase(), 12);
        // no splits remaining
        ticker.increment(610, SECONDS);
        assertEquals(taskExecutor.getRunAwaySplitCount(), 0);
    } finally {
        taskExecutor.stop();
    }
}

Example 15 with TestingTicker

use of com.facebook.airlift.testing.TestingTicker in project presto by prestodb.

the class TestTaskExecutor method testMinMaxDriversPerTask.

@Test(timeOut = 30_000)
public void testMinMaxDriversPerTask() {
    int maxDriversPerTask = 2;
    MultilevelSplitQueue splitQueue = new MultilevelSplitQueue(2);
    TestingTicker ticker = new TestingTicker();
    TaskExecutor taskExecutor = new TaskExecutor(4, 16, 1, maxDriversPerTask, QUERY_FAIR, splitQueue, ticker);
    taskExecutor.start();
    try {
        TaskHandle testTaskHandle = taskExecutor.addTask(new TaskId("test", 0, 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty());
        // enqueue all batches of splits
        int batchCount = 4;
        TestingJob[] splits = new TestingJob[8];
        Phaser[] phasers = new Phaser[batchCount];
        for (int batch = 0; batch < batchCount; batch++) {
            phasers[batch] = new Phaser();
            phasers[batch].register();
            TestingJob split1 = new TestingJob(ticker, new Phaser(), new Phaser(), phasers[batch], 1, 0);
            TestingJob split2 = new TestingJob(ticker, new Phaser(), new Phaser(), phasers[batch], 1, 0);
            splits[2 * batch] = split1;
            splits[2 * batch + 1] = split2;
            taskExecutor.enqueueSplits(testTaskHandle, false, ImmutableList.of(split1, split2));
        }
        // assert that the splits are processed in batches as expected
        for (int batch = 0; batch < batchCount; batch++) {
            // wait until the current batch starts
            waitUntilSplitsStart(ImmutableList.of(splits[2 * batch], splits[2 * batch + 1]));
            // assert that only the splits including and up to the current batch are running and the rest haven't started yet
            assertSplitStates(2 * batch + 1, splits);
            // complete the current batch
            phasers[batch].arriveAndDeregister();
        }
    } finally {
        taskExecutor.stop();
    }
}