Examples with TestingTicker com.facebook.airlift.testing.TestingTicker used on opensource projects

Example 1 with TestingTicker

use of com.facebook.airlift.testing.TestingTicker in project presto by prestodb.

the class TestShardCleaner method setup.

@BeforeMethod
public void setup() {
    dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime() + "_" + ThreadLocalRandom.current().nextInt());
    dummyHandle = dbi.open();
    createTablesWithRetry(dbi);
    temporary = createTempDir();
    File directory = new File(temporary, "data");
    storageService = new LocalFileStorageService(new LocalOrcDataEnvironment(), directory.toURI());
    storageService.start();
    File backupDirectory = new File(temporary, "backup");
    backupStore = new FileBackupStore(backupDirectory);
    ((FileBackupStore) backupStore).start();
    ticker = new TestingTicker();
    ShardCleanerConfig config = new ShardCleanerConfig();
    cleaner = new ShardCleaner(new DaoSupplier<>(dbi, H2ShardDao.class), "node1", true, ticker, storageService, Optional.of(backupStore), new LocalOrcDataEnvironment(), config.getMaxTransactionAge(), config.getTransactionCleanerInterval(), config.getLocalCleanerInterval(), config.getLocalCleanTime(), config.getBackupCleanerInterval(), config.getBackupCleanTime(), config.getBackupDeletionThreads(), config.getMaxCompletedTransactionAge());
}

Example 2 with TestingTicker

use of com.facebook.airlift.testing.TestingTicker in project presto by prestodb.

the class TestTaskExecutor method testLevelMultipliers.

@Test(invocationCount = 100)
public void testLevelMultipliers() throws Exception {
    TestingTicker ticker = new TestingTicker();
    TaskExecutor taskExecutor = new TaskExecutor(1, 3, 3, 4, TASK_FAIR, new MultilevelSplitQueue(2), ticker);
    taskExecutor.start();
    ticker.increment(20, MILLISECONDS);
    try {
        for (int i = 0; i < (LEVEL_THRESHOLD_SECONDS.length - 1); i++) {
            TaskHandle[] taskHandles = { taskExecutor.addTask(new TaskId("test1", 0, 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()), taskExecutor.addTask(new TaskId("test2", 0, 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()), taskExecutor.addTask(new TaskId("test3", 0, 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()) };
            // move task 0 to next level
            TestingJob task0Job = new TestingJob(ticker, new Phaser(1), new Phaser(), new Phaser(), 1, LEVEL_THRESHOLD_SECONDS[i + 1] * 1000);
            taskExecutor.enqueueSplits(taskHandles[0], true, ImmutableList.of(task0Job));
            // move tasks 1 and 2 to this level
            TestingJob task1Job = new TestingJob(ticker, new Phaser(1), new Phaser(), new Phaser(), 1, LEVEL_THRESHOLD_SECONDS[i] * 1000);
            taskExecutor.enqueueSplits(taskHandles[1], true, ImmutableList.of(task1Job));
            TestingJob task2Job = new TestingJob(ticker, new Phaser(1), new Phaser(), new Phaser(), 1, LEVEL_THRESHOLD_SECONDS[i] * 1000);
            taskExecutor.enqueueSplits(taskHandles[2], true, ImmutableList.of(task2Job));
            task0Job.getCompletedFuture().get();
            task1Job.getCompletedFuture().get();
            task2Job.getCompletedFuture().get();
            // then, start new drivers for all tasks
            Phaser globalPhaser = new Phaser(2);
            int phasesForNextLevel = LEVEL_THRESHOLD_SECONDS[i + 1] - LEVEL_THRESHOLD_SECONDS[i];
            TestingJob[] drivers = new TestingJob[6];
            for (int j = 0; j < 6; j++) {
                drivers[j] = new TestingJob(ticker, globalPhaser, new Phaser(), new Phaser(), phasesForNextLevel, 1000);
            }
            taskExecutor.enqueueSplits(taskHandles[0], true, ImmutableList.of(drivers[0], drivers[1]));
            taskExecutor.enqueueSplits(taskHandles[1], true, ImmutableList.of(drivers[2], drivers[3]));
            taskExecutor.enqueueSplits(taskHandles[2], true, ImmutableList.of(drivers[4], drivers[5]));
            // run all three drivers
            int lowerLevelStart = drivers[2].getCompletedPhases() + drivers[3].getCompletedPhases() + drivers[4].getCompletedPhases() + drivers[5].getCompletedPhases();
            int higherLevelStart = drivers[0].getCompletedPhases() + drivers[1].getCompletedPhases();
            while (Arrays.stream(drivers).noneMatch(TestingJob::isFinished)) {
                globalPhaser.arriveAndAwaitAdvance();
                int lowerLevelEnd = drivers[2].getCompletedPhases() + drivers[3].getCompletedPhases() + drivers[4].getCompletedPhases() + drivers[5].getCompletedPhases();
                int lowerLevelTime = lowerLevelEnd - lowerLevelStart;
                int higherLevelEnd = drivers[0].getCompletedPhases() + drivers[1].getCompletedPhases();
                int higherLevelTime = higherLevelEnd - higherLevelStart;
                if (higherLevelTime > 20) {
                    assertGreaterThan(lowerLevelTime, (higherLevelTime * 2) - 10);
                    assertLessThan(higherLevelTime, (lowerLevelTime * 2) + 10);
                }
            }
            try {
                globalPhaser.arriveAndDeregister();
            } catch (IllegalStateException e) {
            // under high concurrency sometimes the deregister call can occur after completion
            // this is not a real problem
            }
            taskExecutor.removeTask(taskHandles[0]);
            taskExecutor.removeTask(taskHandles[1]);
            taskExecutor.removeTask(taskHandles[2]);
        }
    } finally {
        taskExecutor.stop();
    }
}

Example 3 with TestingTicker

use of com.facebook.airlift.testing.TestingTicker in project presto by prestodb.

the class TestTaskExecutor method testQuantaFairness.

@Test(invocationCount = 100)
public void testQuantaFairness() {
    TestingTicker ticker = new TestingTicker();
    TaskExecutor taskExecutor = new TaskExecutor(1, 2, 3, 4, QUERY_FAIR, ticker);
    taskExecutor.start();
    ticker.increment(20, MILLISECONDS);
    try {
        TaskHandle shortQuantaTaskHandle = taskExecutor.addTask(new TaskId("short_quanta", 0, 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty());
        TaskHandle longQuantaTaskHandle = taskExecutor.addTask(new TaskId("long_quanta", 0, 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty());
        Phaser globalPhaser = new Phaser();
        TestingJob shortQuantaDriver = new TestingJob(ticker, new Phaser(), new Phaser(), globalPhaser, 10, 10);
        TestingJob longQuantaDriver = new TestingJob(ticker, new Phaser(), new Phaser(), globalPhaser, 10, 20);
        taskExecutor.enqueueSplits(shortQuantaTaskHandle, true, ImmutableList.of(shortQuantaDriver));
        taskExecutor.enqueueSplits(longQuantaTaskHandle, true, ImmutableList.of(longQuantaDriver));
        for (int i = 0; i < 11; i++) {
            globalPhaser.arriveAndAwaitAdvance();
        }
        assertTrue(shortQuantaDriver.getCompletedPhases() >= 7 && shortQuantaDriver.getCompletedPhases() <= 8);
        assertTrue(longQuantaDriver.getCompletedPhases() >= 3 && longQuantaDriver.getCompletedPhases() <= 4);
        globalPhaser.arriveAndDeregister();
    } finally {
        taskExecutor.stop();
    }
}

Example 4 with TestingTicker

use of com.facebook.airlift.testing.TestingTicker in project presto by prestodb.

the class TestTaskExecutor method testTaskHandle.

@Test
public void testTaskHandle() {
    TestingTicker ticker = new TestingTicker();
    TaskExecutor taskExecutor = new TaskExecutor(4, 8, 3, 4, QUERY_FAIR, ticker);
    taskExecutor.start();
    try {
        TaskId taskId = new TaskId("test", 0, 0, 0);
        TaskHandle taskHandle = taskExecutor.addTask(taskId, () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty());
        Phaser beginPhase = new Phaser();
        beginPhase.register();
        Phaser verificationComplete = new Phaser();
        verificationComplete.register();
        TestingJob driver1 = new TestingJob(ticker, new Phaser(), beginPhase, verificationComplete, 10, 0);
        TestingJob driver2 = new TestingJob(ticker, new Phaser(), beginPhase, verificationComplete, 10, 0);
        // force enqueue a split
        taskExecutor.enqueueSplits(taskHandle, true, ImmutableList.of(driver1));
        assertEquals(taskHandle.getRunningLeafSplits(), 0);
        // normal enqueue a split
        taskExecutor.enqueueSplits(taskHandle, false, ImmutableList.of(driver2));
        assertEquals(taskHandle.getRunningLeafSplits(), 1);
        // let the split continue to run
        beginPhase.arriveAndDeregister();
        verificationComplete.arriveAndDeregister();
    } finally {
        taskExecutor.stop();
    }
}

Example 5 with TestingTicker

use of com.facebook.airlift.testing.TestingTicker in project presto by prestodb.

the class TestPageBufferClient method testExceptionFromResponseHandler.

@Test
public void testExceptionFromResponseHandler() throws Exception {
    DataSize expectedMaxSize = new DataSize(10, Unit.MEGABYTE);
    TestingTicker ticker = new TestingTicker();
    AtomicReference<Duration> tickerIncrement = new AtomicReference<>(new Duration(0, TimeUnit.SECONDS));
    TestingHttpClient.Processor processor = (input) -> {
        Duration delta = tickerIncrement.get();
        ticker.increment(delta.toMillis(), TimeUnit.MILLISECONDS);
        throw new RuntimeException("Foo");
    };
    CyclicBarrier requestComplete = new CyclicBarrier(2);
    TestingClientCallback callback = new TestingClientCallback(requestComplete);
    URI location = URI.create("http://localhost:8080");
    PageBufferClient client = new PageBufferClient(new HttpRpcShuffleClient(new TestingHttpClient(processor, scheduler), location), new Duration(30, TimeUnit.SECONDS), true, location, Optional.empty(), callback, scheduler, ticker, pageBufferClientCallbackExecutor);
    assertStatus(client, location, "queued", 0, 0, 0, 0, "not scheduled");
    // request processor will throw exception, verify the request is marked a completed
    // this starts the error stopwatch
    client.scheduleRequest(expectedMaxSize);
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 1);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 0);
    assertStatus(client, location, "queued", 0, 1, 1, 1, "not scheduled");
    // advance time forward, but not enough to fail the client
    tickerIncrement.set(new Duration(30, TimeUnit.SECONDS));
    // verify that the client has not failed
    client.scheduleRequest(expectedMaxSize);
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 2);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 0);
    assertStatus(client, location, "queued", 0, 2, 2, 2, "not scheduled");
    // advance time forward beyond the minimum error duration
    tickerIncrement.set(new Duration(31, TimeUnit.SECONDS));
    // verify that the client has failed
    client.scheduleRequest(expectedMaxSize);
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 3);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 1);
    assertInstanceOf(callback.getFailure(), PageTransportTimeoutException.class);
    assertContains(callback.getFailure().getMessage(), WORKER_NODE_ERROR + " (http://localhost:8080/0 - 3 failures, failure duration 31.00s, total failed request time 31.00s)");
    assertStatus(client, location, "queued", 0, 3, 3, 3, "not scheduled");
}