Search in sources :

Example 1 with TestingTicker

use of io.airlift.testing.TestingTicker in project presto by prestodb.

the class TestTaskExecutor method test.

@Test(invocationCount = 100)
public void test() throws Exception {
    TestingTicker ticker = new TestingTicker();
    TaskExecutor taskExecutor = new TaskExecutor(4, 8, ticker);
    taskExecutor.start();
    ticker.increment(20, MILLISECONDS);
    try {
        TaskId taskId = new TaskId("test", 0, 0);
        TaskHandle taskHandle = taskExecutor.addTask(taskId, () -> 0, 10, new Duration(1, MILLISECONDS));
        Phaser beginPhase = new Phaser();
        beginPhase.register();
        Phaser verificationComplete = new Phaser();
        verificationComplete.register();
        // add two jobs
        TestingJob driver1 = new TestingJob(beginPhase, verificationComplete, 10);
        ListenableFuture<?> future1 = getOnlyElement(taskExecutor.enqueueSplits(taskHandle, true, ImmutableList.of(driver1)));
        TestingJob driver2 = new TestingJob(beginPhase, verificationComplete, 10);
        ListenableFuture<?> future2 = getOnlyElement(taskExecutor.enqueueSplits(taskHandle, true, ImmutableList.of(driver2)));
        assertEquals(driver1.getCompletedPhases(), 0);
        assertEquals(driver2.getCompletedPhases(), 0);
        // verify worker have arrived but haven't processed yet
        beginPhase.arriveAndAwaitAdvance();
        assertEquals(driver1.getCompletedPhases(), 0);
        assertEquals(driver2.getCompletedPhases(), 0);
        ticker.increment(10, MILLISECONDS);
        assertEquals(taskExecutor.getMaxActiveSplitTime(), 10);
        verificationComplete.arriveAndAwaitAdvance();
        // advance one phase and verify
        beginPhase.arriveAndAwaitAdvance();
        assertEquals(driver1.getCompletedPhases(), 1);
        assertEquals(driver2.getCompletedPhases(), 1);
        verificationComplete.arriveAndAwaitAdvance();
        // add one more job
        TestingJob driver3 = new TestingJob(beginPhase, verificationComplete, 10);
        ListenableFuture<?> future3 = getOnlyElement(taskExecutor.enqueueSplits(taskHandle, false, ImmutableList.of(driver3)));
        // advance one phase and verify
        beginPhase.arriveAndAwaitAdvance();
        assertEquals(driver1.getCompletedPhases(), 2);
        assertEquals(driver2.getCompletedPhases(), 2);
        assertEquals(driver3.getCompletedPhases(), 0);
        verificationComplete.arriveAndAwaitAdvance();
        // advance to the end of the first two task and verify
        beginPhase.arriveAndAwaitAdvance();
        for (int i = 0; i < 7; i++) {
            verificationComplete.arriveAndAwaitAdvance();
            beginPhase.arriveAndAwaitAdvance();
            assertEquals(beginPhase.getPhase(), verificationComplete.getPhase() + 1);
        }
        assertEquals(driver1.getCompletedPhases(), 10);
        assertEquals(driver2.getCompletedPhases(), 10);
        assertEquals(driver3.getCompletedPhases(), 8);
        future1.get(1, TimeUnit.SECONDS);
        future2.get(1, TimeUnit.SECONDS);
        verificationComplete.arriveAndAwaitAdvance();
        // advance two more times and verify
        beginPhase.arriveAndAwaitAdvance();
        verificationComplete.arriveAndAwaitAdvance();
        beginPhase.arriveAndAwaitAdvance();
        assertEquals(driver1.getCompletedPhases(), 10);
        assertEquals(driver2.getCompletedPhases(), 10);
        assertEquals(driver3.getCompletedPhases(), 10);
        future3.get(1, TimeUnit.SECONDS);
        verificationComplete.arriveAndAwaitAdvance();
        assertEquals(driver1.getFirstPhase(), 0);
        assertEquals(driver2.getFirstPhase(), 0);
        assertEquals(driver3.getFirstPhase(), 2);
        assertEquals(driver1.getLastPhase(), 10);
        assertEquals(driver2.getLastPhase(), 10);
        assertEquals(driver3.getLastPhase(), 12);
        // no splits remaining
        ticker.increment(30, MILLISECONDS);
        assertEquals(taskExecutor.getMaxActiveSplitTime(), 0);
    } finally {
        taskExecutor.stop();
    }
}
Also used : TestingTicker(io.airlift.testing.TestingTicker) Duration(io.airlift.units.Duration) Phaser(java.util.concurrent.Phaser) TaskHandle(com.facebook.presto.execution.TaskExecutor.TaskHandle) Test(org.testng.annotations.Test)

Example 2 with TestingTicker

use of io.airlift.testing.TestingTicker in project presto by prestodb.

the class TestHttpPageBufferClient method testExceptionFromResponseHandler.

@Test
public void testExceptionFromResponseHandler() throws Exception {
    TestingTicker ticker = new TestingTicker();
    AtomicReference<Duration> tickerIncrement = new AtomicReference<>(new Duration(0, TimeUnit.SECONDS));
    TestingHttpClient.Processor processor = (input) -> {
        Duration delta = tickerIncrement.get();
        ticker.increment(delta.toMillis(), TimeUnit.MILLISECONDS);
        throw new RuntimeException("Foo");
    };
    CyclicBarrier requestComplete = new CyclicBarrier(2);
    TestingClientCallback callback = new TestingClientCallback(requestComplete);
    URI location = URI.create("http://localhost:8080");
    HttpPageBufferClient client = new HttpPageBufferClient(new TestingHttpClient(processor, executor), new DataSize(10, Unit.MEGABYTE), new Duration(1, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), location, callback, executor, ticker);
    assertStatus(client, location, "queued", 0, 0, 0, 0, "not scheduled");
    // request processor will throw exception, verify the request is marked a completed
    // this starts the error stopwatch
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 1);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 0);
    assertStatus(client, location, "queued", 0, 1, 1, 1, "not scheduled");
    // advance time forward, but not enough to fail the client
    tickerIncrement.set(new Duration(30, TimeUnit.SECONDS));
    // verify that the client has not failed
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 2);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 0);
    assertStatus(client, location, "queued", 0, 2, 2, 2, "not scheduled");
    // advance time forward beyond the minimum error duration
    tickerIncrement.set(new Duration(31, TimeUnit.SECONDS));
    // verify that the client has failed
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 3);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 1);
    assertInstanceOf(callback.getFailure(), PageTransportTimeoutException.class);
    assertContains(callback.getFailure().getMessage(), WORKER_NODE_ERROR + " (http://localhost:8080/0 - 3 failures, time since last success 61.00s)");
    assertStatus(client, location, "queued", 0, 3, 3, 3, "not scheduled");
}
Also used : Page(com.facebook.presto.spi.Page) TestingTicker(io.airlift.testing.TestingTicker) TestingPagesSerdeFactory.testingPagesSerde(com.facebook.presto.execution.buffer.TestingPagesSerdeFactory.testingPagesSerde) ClientCallback(com.facebook.presto.operator.HttpPageBufferClient.ClientCallback) Assertions.assertContains(io.airlift.testing.Assertions.assertContains) Assertions.assertInstanceOf(io.airlift.testing.Assertions.assertInstanceOf) TimeoutException(java.util.concurrent.TimeoutException) Assert.assertEquals(org.testng.Assert.assertEquals) Test(org.testng.annotations.Test) Unit(io.airlift.units.DataSize.Unit) AtomicReference(java.util.concurrent.atomic.AtomicReference) CONTENT_TYPE(com.google.common.net.HttpHeaders.CONTENT_TYPE) Duration(io.airlift.units.Duration) ArrayList(java.util.ArrayList) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Executors.newScheduledThreadPool(java.util.concurrent.Executors.newScheduledThreadPool) PAGE_TOO_LARGE(com.facebook.presto.spi.StandardErrorCode.PAGE_TOO_LARGE) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) Request(io.airlift.http.client.Request) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) TestingResponse(io.airlift.http.client.testing.TestingResponse) URI(java.net.URI) PAGE_TRANSPORT_ERROR(com.facebook.presto.spi.StandardErrorCode.PAGE_TRANSPORT_ERROR) AfterClass(org.testng.annotations.AfterClass) CyclicBarrier(java.util.concurrent.CyclicBarrier) BeforeClass(org.testng.annotations.BeforeClass) TestingHttpClient(io.airlift.http.client.testing.TestingHttpClient) Throwables(com.google.common.base.Throwables) BrokenBarrierException(java.util.concurrent.BrokenBarrierException) PAGE_TRANSPORT_TIMEOUT(com.facebook.presto.spi.StandardErrorCode.PAGE_TRANSPORT_TIMEOUT) WORKER_NODE_ERROR(com.facebook.presto.util.Failures.WORKER_NODE_ERROR) Collectors(java.util.stream.Collectors) TimeUnit(java.util.concurrent.TimeUnit) DataSize(io.airlift.units.DataSize) List(java.util.List) PRESTO_PAGES(com.facebook.presto.PrestoMediaTypes.PRESTO_PAGES) HttpStatus(io.airlift.http.client.HttpStatus) SerializedPage(com.facebook.presto.execution.buffer.SerializedPage) ImmutableListMultimap(com.google.common.collect.ImmutableListMultimap) Response(io.airlift.http.client.Response) PagesSerde(com.facebook.presto.execution.buffer.PagesSerde) Collections(java.util.Collections) TestingTicker(io.airlift.testing.TestingTicker) AtomicReference(java.util.concurrent.atomic.AtomicReference) Duration(io.airlift.units.Duration) URI(java.net.URI) CyclicBarrier(java.util.concurrent.CyclicBarrier) TestingHttpClient(io.airlift.http.client.testing.TestingHttpClient) DataSize(io.airlift.units.DataSize) Test(org.testng.annotations.Test)

Example 3 with TestingTicker

use of io.airlift.testing.TestingTicker in project airlift by airlift.

the class TestDecayTDigest method testDecayBeyondRescaleThreshold.

@Test
public void testDecayBeyondRescaleThreshold() {
    TestingTicker ticker = new TestingTicker();
    double decayFactor = 0.1;
    DecayTDigest digest = new DecayTDigest(100, decayFactor, ticker);
    // the weight of this value should decay to 1 after time advances by deltaTime
    digest.add(1, Math.exp(decayFactor * RESCALE_THRESHOLD_SECONDS));
    // advancing the time by this amount will trigger a rescale
    ticker.increment(RESCALE_THRESHOLD_SECONDS, TimeUnit.SECONDS);
    assertThat(digest.getCount()).isCloseTo(1.0, Offset.offset(ZERO_WEIGHT_THRESHOLD));
    digest.add(2);
    assertThat(digest.getCount()).isCloseTo(2.0, Offset.offset(ZERO_WEIGHT_THRESHOLD));
}
Also used : TestingTicker(io.airlift.testing.TestingTicker) Test(org.testng.annotations.Test)

Example 4 with TestingTicker

use of io.airlift.testing.TestingTicker in project airlift by airlift.

the class TestQuantileDigest method testMinMax.

@Test
public void testMinMax() throws Exception {
    QuantileDigest digest = new QuantileDigest(0.01, 0, new TestingTicker());
    int from = 500;
    int to = 700;
    addRange(digest, from, to + 1);
    assertEquals(digest.getMin(), from);
    assertEquals(digest.getMax(), to);
}
Also used : TestingTicker(io.airlift.testing.TestingTicker) Test(org.testng.annotations.Test)

Example 5 with TestingTicker

use of io.airlift.testing.TestingTicker in project airlift by airlift.

the class TestQuantileDigest method testCompression.

@Test
public void testCompression() throws Exception {
    QuantileDigest digest = new QuantileDigest(1, 0, new TestingTicker());
    for (int loop = 0; loop < 2; ++loop) {
        addRange(digest, 0, 15);
        digest.compress();
        digest.validate();
    }
}
Also used : TestingTicker(io.airlift.testing.TestingTicker) Test(org.testng.annotations.Test)

Aggregations

TestingTicker (io.airlift.testing.TestingTicker)17 Test (org.testng.annotations.Test)17 Duration (io.airlift.units.Duration)3 PRESTO_PAGES (com.facebook.presto.PrestoMediaTypes.PRESTO_PAGES)1 TaskHandle (com.facebook.presto.execution.TaskExecutor.TaskHandle)1 PagesSerde (com.facebook.presto.execution.buffer.PagesSerde)1 SerializedPage (com.facebook.presto.execution.buffer.SerializedPage)1 TestingPagesSerdeFactory.testingPagesSerde (com.facebook.presto.execution.buffer.TestingPagesSerdeFactory.testingPagesSerde)1 ClientCallback (com.facebook.presto.operator.HttpPageBufferClient.ClientCallback)1 Page (com.facebook.presto.spi.Page)1 PAGE_TOO_LARGE (com.facebook.presto.spi.StandardErrorCode.PAGE_TOO_LARGE)1 PAGE_TRANSPORT_ERROR (com.facebook.presto.spi.StandardErrorCode.PAGE_TRANSPORT_ERROR)1 PAGE_TRANSPORT_TIMEOUT (com.facebook.presto.spi.StandardErrorCode.PAGE_TRANSPORT_TIMEOUT)1 WORKER_NODE_ERROR (com.facebook.presto.util.Failures.WORKER_NODE_ERROR)1 Throwables (com.google.common.base.Throwables)1 ImmutableListMultimap (com.google.common.collect.ImmutableListMultimap)1 CONTENT_TYPE (com.google.common.net.HttpHeaders.CONTENT_TYPE)1 Threads.daemonThreadsNamed (io.airlift.concurrent.Threads.daemonThreadsNamed)1 HttpStatus (io.airlift.http.client.HttpStatus)1 Request (io.airlift.http.client.Request)1