Search in sources :

Example 1 with TestingHttpClient

use of io.airlift.http.client.testing.TestingHttpClient in project presto by prestodb.

the class TestProgressMonitor method createConnection.

private Connection createConnection() throws SQLException {
    HttpClient client = new TestingHttpClient(new TestingHttpClientProcessor(RESPONSES));
    QueryExecutor testQueryExecutor = QueryExecutor.create(client);
    String uri = format("prestotest://%s", SERVER_ADDRESS);
    return new PrestoConnection(new PrestoDriverUri(uri), "test", testQueryExecutor);
}
Also used : HttpClient(io.airlift.http.client.HttpClient) TestingHttpClient(io.airlift.http.client.testing.TestingHttpClient) TestingHttpClient(io.airlift.http.client.testing.TestingHttpClient)

Example 2 with TestingHttpClient

use of io.airlift.http.client.testing.TestingHttpClient in project presto by prestodb.

the class TestExchangeClient method testHappyPath.

@Test
public void testHappyPath() throws Exception {
    DataSize maxResponseSize = new DataSize(10, Unit.MEGABYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);
    URI location = URI.create("http://localhost:8080");
    processor.addPage(location, createPage(1));
    processor.addPage(location, createPage(2));
    processor.addPage(location, createPage(3));
    processor.setComplete(location);
    @SuppressWarnings("resource") ExchangeClient exchangeClient = new ExchangeClient(new DataSize(32, Unit.MEGABYTE), maxResponseSize, 1, new Duration(1, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), new TestingHttpClient(processor, executor), executor, deltaMemoryInBytes -> {
    });
    exchangeClient.addLocation(location);
    exchangeClient.noMoreLocations();
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(1));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(2));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(3));
    assertNull(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)));
    assertEquals(exchangeClient.isClosed(), true);
    if (exchangeClient.getStatus().getBufferedPages() != 0) {
        assertEquals(exchangeClient.getStatus().getBufferedPages(), 0);
    }
    assertTrue(exchangeClient.getStatus().getBufferedBytes() == 0);
    // client should have sent only 2 requests: one to get all pages and once to get the done signal
    assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "closed", 3, 3, 3, "not scheduled");
}
Also used : DataSize(io.airlift.units.DataSize) TestingHttpClient(io.airlift.http.client.testing.TestingHttpClient) Duration(io.airlift.units.Duration) URI(java.net.URI) Test(org.testng.annotations.Test)

Example 3 with TestingHttpClient

use of io.airlift.http.client.testing.TestingHttpClient in project presto by prestodb.

the class TestHttpPageBufferClient method testExceptionFromResponseHandler.

@Test
public void testExceptionFromResponseHandler() throws Exception {
    TestingTicker ticker = new TestingTicker();
    AtomicReference<Duration> tickerIncrement = new AtomicReference<>(new Duration(0, TimeUnit.SECONDS));
    TestingHttpClient.Processor processor = (input) -> {
        Duration delta = tickerIncrement.get();
        ticker.increment(delta.toMillis(), TimeUnit.MILLISECONDS);
        throw new RuntimeException("Foo");
    };
    CyclicBarrier requestComplete = new CyclicBarrier(2);
    TestingClientCallback callback = new TestingClientCallback(requestComplete);
    URI location = URI.create("http://localhost:8080");
    HttpPageBufferClient client = new HttpPageBufferClient(new TestingHttpClient(processor, executor), new DataSize(10, Unit.MEGABYTE), new Duration(1, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), location, callback, executor, ticker);
    assertStatus(client, location, "queued", 0, 0, 0, 0, "not scheduled");
    // request processor will throw exception, verify the request is marked a completed
    // this starts the error stopwatch
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 1);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 0);
    assertStatus(client, location, "queued", 0, 1, 1, 1, "not scheduled");
    // advance time forward, but not enough to fail the client
    tickerIncrement.set(new Duration(30, TimeUnit.SECONDS));
    // verify that the client has not failed
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 2);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 0);
    assertStatus(client, location, "queued", 0, 2, 2, 2, "not scheduled");
    // advance time forward beyond the minimum error duration
    tickerIncrement.set(new Duration(31, TimeUnit.SECONDS));
    // verify that the client has failed
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 3);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 1);
    assertInstanceOf(callback.getFailure(), PageTransportTimeoutException.class);
    assertContains(callback.getFailure().getMessage(), WORKER_NODE_ERROR + " (http://localhost:8080/0 - 3 failures, time since last success 61.00s)");
    assertStatus(client, location, "queued", 0, 3, 3, 3, "not scheduled");
}
Also used : Page(com.facebook.presto.spi.Page) TestingTicker(io.airlift.testing.TestingTicker) TestingPagesSerdeFactory.testingPagesSerde(com.facebook.presto.execution.buffer.TestingPagesSerdeFactory.testingPagesSerde) ClientCallback(com.facebook.presto.operator.HttpPageBufferClient.ClientCallback) Assertions.assertContains(io.airlift.testing.Assertions.assertContains) Assertions.assertInstanceOf(io.airlift.testing.Assertions.assertInstanceOf) TimeoutException(java.util.concurrent.TimeoutException) Assert.assertEquals(org.testng.Assert.assertEquals) Test(org.testng.annotations.Test) Unit(io.airlift.units.DataSize.Unit) AtomicReference(java.util.concurrent.atomic.AtomicReference) CONTENT_TYPE(com.google.common.net.HttpHeaders.CONTENT_TYPE) Duration(io.airlift.units.Duration) ArrayList(java.util.ArrayList) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Executors.newScheduledThreadPool(java.util.concurrent.Executors.newScheduledThreadPool) PAGE_TOO_LARGE(com.facebook.presto.spi.StandardErrorCode.PAGE_TOO_LARGE) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) Request(io.airlift.http.client.Request) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) TestingResponse(io.airlift.http.client.testing.TestingResponse) URI(java.net.URI) PAGE_TRANSPORT_ERROR(com.facebook.presto.spi.StandardErrorCode.PAGE_TRANSPORT_ERROR) AfterClass(org.testng.annotations.AfterClass) CyclicBarrier(java.util.concurrent.CyclicBarrier) BeforeClass(org.testng.annotations.BeforeClass) TestingHttpClient(io.airlift.http.client.testing.TestingHttpClient) Throwables(com.google.common.base.Throwables) BrokenBarrierException(java.util.concurrent.BrokenBarrierException) PAGE_TRANSPORT_TIMEOUT(com.facebook.presto.spi.StandardErrorCode.PAGE_TRANSPORT_TIMEOUT) WORKER_NODE_ERROR(com.facebook.presto.util.Failures.WORKER_NODE_ERROR) Collectors(java.util.stream.Collectors) TimeUnit(java.util.concurrent.TimeUnit) DataSize(io.airlift.units.DataSize) List(java.util.List) PRESTO_PAGES(com.facebook.presto.PrestoMediaTypes.PRESTO_PAGES) HttpStatus(io.airlift.http.client.HttpStatus) SerializedPage(com.facebook.presto.execution.buffer.SerializedPage) ImmutableListMultimap(com.google.common.collect.ImmutableListMultimap) Response(io.airlift.http.client.Response) PagesSerde(com.facebook.presto.execution.buffer.PagesSerde) Collections(java.util.Collections) TestingTicker(io.airlift.testing.TestingTicker) AtomicReference(java.util.concurrent.atomic.AtomicReference) Duration(io.airlift.units.Duration) URI(java.net.URI) CyclicBarrier(java.util.concurrent.CyclicBarrier) TestingHttpClient(io.airlift.http.client.testing.TestingHttpClient) DataSize(io.airlift.units.DataSize) Test(org.testng.annotations.Test)

Example 4 with TestingHttpClient

use of io.airlift.http.client.testing.TestingHttpClient in project presto by prestodb.

the class TestHttpPageBufferClient method testHappyPath.

@Test
public void testHappyPath() throws Exception {
    Page expectedPage = new Page(100);
    DataSize expectedMaxSize = new DataSize(11, Unit.MEGABYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(expectedMaxSize);
    CyclicBarrier requestComplete = new CyclicBarrier(2);
    TestingClientCallback callback = new TestingClientCallback(requestComplete);
    URI location = URI.create("http://localhost:8080");
    HttpPageBufferClient client = new HttpPageBufferClient(new TestingHttpClient(processor, executor), expectedMaxSize, new Duration(1, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), location, callback, executor);
    assertStatus(client, location, "queued", 0, 0, 0, 0, "not scheduled");
    // fetch a page and verify
    processor.addPage(location, expectedPage);
    callback.resetStats();
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 1);
    assertPageEquals(expectedPage, callback.getPages().get(0));
    assertEquals(callback.getCompletedRequests(), 1);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertStatus(client, location, "queued", 1, 1, 1, 0, "not scheduled");
    // fetch no data and verify
    callback.resetStats();
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 1);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertStatus(client, location, "queued", 1, 2, 2, 0, "not scheduled");
    // fetch two more pages and verify
    processor.addPage(location, expectedPage);
    processor.addPage(location, expectedPage);
    callback.resetStats();
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 2);
    assertPageEquals(expectedPage, callback.getPages().get(0));
    assertPageEquals(expectedPage, callback.getPages().get(1));
    assertEquals(callback.getCompletedRequests(), 1);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 0);
    callback.resetStats();
    assertStatus(client, location, "queued", 3, 3, 3, 0, "not scheduled");
    // finish and verify
    callback.resetStats();
    processor.setComplete(location);
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    // get the buffer complete signal
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 1);
    // schedule the delete call to the buffer
    callback.resetStats();
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getFinishedBuffers(), 1);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 0);
    assertEquals(callback.getFailedBuffers(), 0);
    assertStatus(client, location, "closed", 3, 5, 5, 0, "not scheduled");
}
Also used : DataSize(io.airlift.units.DataSize) TestingHttpClient(io.airlift.http.client.testing.TestingHttpClient) Page(com.facebook.presto.spi.Page) SerializedPage(com.facebook.presto.execution.buffer.SerializedPage) Duration(io.airlift.units.Duration) URI(java.net.URI) CyclicBarrier(java.util.concurrent.CyclicBarrier) Test(org.testng.annotations.Test)

Example 5 with TestingHttpClient

use of io.airlift.http.client.testing.TestingHttpClient in project presto by prestodb.

the class TestHttpPageBufferClient method testCloseDuringPendingRequest.

@Test
public void testCloseDuringPendingRequest() throws Exception {
    CyclicBarrier beforeRequest = new CyclicBarrier(2);
    CyclicBarrier afterRequest = new CyclicBarrier(2);
    StaticRequestProcessor processor = new StaticRequestProcessor(beforeRequest, afterRequest);
    processor.setResponse(new TestingResponse(HttpStatus.NO_CONTENT, ImmutableListMultimap.of(), new byte[0]));
    CyclicBarrier requestComplete = new CyclicBarrier(2);
    TestingClientCallback callback = new TestingClientCallback(requestComplete);
    URI location = URI.create("http://localhost:8080");
    HttpPageBufferClient client = new HttpPageBufferClient(new TestingHttpClient(processor, executor), new DataSize(10, Unit.MEGABYTE), new Duration(1, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), location, callback, executor);
    assertStatus(client, location, "queued", 0, 0, 0, 0, "not scheduled");
    // send request
    client.scheduleRequest();
    beforeRequest.await(10, TimeUnit.SECONDS);
    assertStatus(client, location, "running", 0, 1, 0, 0, "PROCESSING_REQUEST");
    assertEquals(client.isRunning(), true);
    // request is pending, now close it
    client.close();
    try {
        requestComplete.await(10, TimeUnit.SECONDS);
    } catch (BrokenBarrierException ignored) {
    }
    try {
        afterRequest.await(10, TimeUnit.SECONDS);
    } catch (BrokenBarrierException ignored) {
        afterRequest.reset();
    }
    // client.close() triggers a DELETE request, so wait for it to finish
    beforeRequest.await(10, TimeUnit.SECONDS);
    afterRequest.await(10, TimeUnit.SECONDS);
    requestComplete.await(10, TimeUnit.SECONDS);
    assertStatus(client, location, "closed", 0, 1, 2, 1, "not scheduled");
}
Also used : TestingResponse(io.airlift.http.client.testing.TestingResponse) BrokenBarrierException(java.util.concurrent.BrokenBarrierException) TestingHttpClient(io.airlift.http.client.testing.TestingHttpClient) DataSize(io.airlift.units.DataSize) Duration(io.airlift.units.Duration) URI(java.net.URI) CyclicBarrier(java.util.concurrent.CyclicBarrier) Test(org.testng.annotations.Test)

Aggregations

TestingHttpClient (io.airlift.http.client.testing.TestingHttpClient)8 Duration (io.airlift.units.Duration)7 URI (java.net.URI)7 Test (org.testng.annotations.Test)7 DataSize (io.airlift.units.DataSize)6 TestingResponse (io.airlift.http.client.testing.TestingResponse)5 CyclicBarrier (java.util.concurrent.CyclicBarrier)5 SerializedPage (com.facebook.presto.execution.buffer.SerializedPage)2 Page (com.facebook.presto.spi.Page)2 ImmutableListMultimap (com.google.common.collect.ImmutableListMultimap)2 HttpStatus (io.airlift.http.client.HttpStatus)2 Request (io.airlift.http.client.Request)2 Response (io.airlift.http.client.Response)2 BrokenBarrierException (java.util.concurrent.BrokenBarrierException)2 OutputBuffers (com.facebook.presto.OutputBuffers)1 OutputBuffers.createInitialEmptyOutputBuffers (com.facebook.presto.OutputBuffers.createInitialEmptyOutputBuffers)1 PRESTO_PAGES (com.facebook.presto.PrestoMediaTypes.PRESTO_PAGES)1 TEST_SESSION (com.facebook.presto.SessionTestUtils.TEST_SESSION)1 NodeVersion (com.facebook.presto.client.NodeVersion)1 PRESTO_MAX_WAIT (com.facebook.presto.client.PrestoHeaders.PRESTO_MAX_WAIT)1