use of io.airlift.units.DataSize in project presto by prestodb.
the class TestDriverStats method assertExpectedDriverStats.
public static void assertExpectedDriverStats(DriverStats actual) {
assertEquals(actual.getCreateTime(), new DateTime(1, UTC));
assertEquals(actual.getStartTime(), new DateTime(2, UTC));
assertEquals(actual.getEndTime(), new DateTime(3, UTC));
assertEquals(actual.getQueuedTime(), new Duration(4, NANOSECONDS));
assertEquals(actual.getElapsedTime(), new Duration(5, NANOSECONDS));
assertEquals(actual.getMemoryReservation(), new DataSize(6, BYTE));
assertEquals(actual.getPeakMemoryReservation(), new DataSize(19, BYTE));
assertEquals(actual.getSystemMemoryReservation(), new DataSize(7, BYTE));
assertEquals(actual.getTotalScheduledTime(), new Duration(8, NANOSECONDS));
assertEquals(actual.getTotalCpuTime(), new Duration(9, NANOSECONDS));
assertEquals(actual.getTotalUserTime(), new Duration(10, NANOSECONDS));
assertEquals(actual.getTotalBlockedTime(), new Duration(11, NANOSECONDS));
assertEquals(actual.getRawInputDataSize(), new DataSize(12, BYTE));
assertEquals(actual.getRawInputPositions(), 13);
assertEquals(actual.getRawInputReadTime(), new Duration(14, NANOSECONDS));
assertEquals(actual.getProcessedInputDataSize(), new DataSize(15, BYTE));
assertEquals(actual.getProcessedInputPositions(), 16);
assertEquals(actual.getOutputDataSize(), new DataSize(17, BYTE));
assertEquals(actual.getOutputPositions(), 18);
assertEquals(actual.getOperatorStats().size(), 1);
assertExpectedOperatorStats(actual.getOperatorStats().get(0));
}
use of io.airlift.units.DataSize in project presto by prestodb.
the class TestExchangeClient method testBufferLimit.
@Test
public void testBufferLimit() throws Exception {
DataSize maxResponseSize = new DataSize(1, Unit.BYTE);
MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);
URI location = URI.create("http://localhost:8080");
// add a pages
processor.addPage(location, createPage(1));
processor.addPage(location, createPage(2));
processor.addPage(location, createPage(3));
processor.setComplete(location);
@SuppressWarnings("resource") ExchangeClient exchangeClient = new ExchangeClient(new DataSize(1, Unit.BYTE), maxResponseSize, 1, new Duration(1, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), new TestingHttpClient(processor, newCachedThreadPool(daemonThreadsNamed("test-%s"))), executor, deltaMemoryInBytes -> {
});
exchangeClient.addLocation(location);
exchangeClient.noMoreLocations();
assertEquals(exchangeClient.isClosed(), false);
long start = System.nanoTime();
// start fetching pages
exchangeClient.scheduleRequestIfNecessary();
// wait for a page to be fetched
do {
// there is no thread coordination here, so sleep is the best we can do
assertLessThan(Duration.nanosSince(start), new Duration(5, TimeUnit.SECONDS));
sleepUninterruptibly(100, MILLISECONDS);
} while (exchangeClient.getStatus().getBufferedPages() == 0);
// client should have sent a single request for a single page
assertEquals(exchangeClient.getStatus().getBufferedPages(), 1);
assertTrue(exchangeClient.getStatus().getBufferedBytes() > 0);
assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "queued", 1, 1, 1, "not scheduled");
// remove the page and wait for the client to fetch another page
assertPageEquals(exchangeClient.getNextPage(new Duration(0, TimeUnit.SECONDS)), createPage(1));
do {
assertLessThan(Duration.nanosSince(start), new Duration(5, TimeUnit.SECONDS));
sleepUninterruptibly(100, MILLISECONDS);
} while (exchangeClient.getStatus().getBufferedPages() == 0);
// client should have sent a single request for a single page
assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "queued", 2, 2, 2, "not scheduled");
assertEquals(exchangeClient.getStatus().getBufferedPages(), 1);
assertTrue(exchangeClient.getStatus().getBufferedBytes() > 0);
// remove the page and wait for the client to fetch another page
assertPageEquals(exchangeClient.getNextPage(new Duration(0, TimeUnit.SECONDS)), createPage(2));
do {
assertLessThan(Duration.nanosSince(start), new Duration(5, TimeUnit.SECONDS));
sleepUninterruptibly(100, MILLISECONDS);
} while (exchangeClient.getStatus().getBufferedPages() == 0);
// client should have sent a single request for a single page
assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "queued", 3, 3, 3, "not scheduled");
assertEquals(exchangeClient.getStatus().getBufferedPages(), 1);
assertTrue(exchangeClient.getStatus().getBufferedBytes() > 0);
// remove last page
assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(3));
// wait for client to decide there are no more pages
assertNull(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)));
assertEquals(exchangeClient.getStatus().getBufferedPages(), 0);
assertTrue(exchangeClient.getStatus().getBufferedBytes() == 0);
assertEquals(exchangeClient.isClosed(), true);
assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "closed", 3, 5, 5, "not scheduled");
}
use of io.airlift.units.DataSize in project presto by prestodb.
the class TestExchangeClient method testAddLocation.
@Test(timeOut = 10000)
public void testAddLocation() throws Exception {
DataSize maxResponseSize = new DataSize(10, Unit.MEGABYTE);
MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);
@SuppressWarnings("resource") ExchangeClient exchangeClient = new ExchangeClient(new DataSize(32, Unit.MEGABYTE), maxResponseSize, 1, new Duration(1, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), new TestingHttpClient(processor, newCachedThreadPool(daemonThreadsNamed("test-%s"))), executor, deltaMemoryInBytes -> {
});
URI location1 = URI.create("http://localhost:8081/foo");
processor.addPage(location1, createPage(1));
processor.addPage(location1, createPage(2));
processor.addPage(location1, createPage(3));
processor.setComplete(location1);
exchangeClient.addLocation(location1);
assertEquals(exchangeClient.isClosed(), false);
assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(1));
assertEquals(exchangeClient.isClosed(), false);
assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(2));
assertEquals(exchangeClient.isClosed(), false);
assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(3));
assertNull(exchangeClient.getNextPage(new Duration(10, MILLISECONDS)));
assertEquals(exchangeClient.isClosed(), false);
URI location2 = URI.create("http://localhost:8082/bar");
processor.addPage(location2, createPage(4));
processor.addPage(location2, createPage(5));
processor.addPage(location2, createPage(6));
processor.setComplete(location2);
exchangeClient.addLocation(location2);
assertEquals(exchangeClient.isClosed(), false);
assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(4));
assertEquals(exchangeClient.isClosed(), false);
assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(5));
assertEquals(exchangeClient.isClosed(), false);
assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(6));
assertNull(exchangeClient.getNextPage(new Duration(10, MILLISECONDS)));
assertEquals(exchangeClient.isClosed(), false);
exchangeClient.noMoreLocations();
// receive a final GONE response, so just spin until it's closed or the test times out.
while (!exchangeClient.isClosed()) {
Thread.sleep(1);
}
ImmutableMap<URI, PageBufferClientStatus> statuses = uniqueIndex(exchangeClient.getStatus().getPageBufferClientStatuses(), PageBufferClientStatus::getUri);
assertStatus(statuses.get(location1), location1, "closed", 3, 3, 3, "not scheduled");
assertStatus(statuses.get(location2), location2, "closed", 3, 3, 3, "not scheduled");
}
use of io.airlift.units.DataSize in project presto by prestodb.
the class TestExchangeOperator method setUp.
@SuppressWarnings("resource")
@BeforeClass
public void setUp() throws Exception {
executor = newScheduledThreadPool(4, daemonThreadsNamed("test-%s"));
httpClient = new TestingHttpClient(new HttpClientHandler(taskBuffers), executor);
exchangeClientSupplier = (systemMemoryUsageListener) -> new ExchangeClient(new DataSize(32, MEGABYTE), new DataSize(10, MEGABYTE), 3, new Duration(1, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), httpClient, executor, systemMemoryUsageListener);
}
use of io.airlift.units.DataSize in project presto by prestodb.
the class TestOperatorStats method testAddMergeable.
@Test
public void testAddMergeable() {
OperatorStats actual = MERGEABLE.add(MERGEABLE, MERGEABLE);
assertEquals(actual.getOperatorId(), 41);
assertEquals(actual.getOperatorType(), "test");
assertEquals(actual.getTotalDrivers(), 3 * 1);
assertEquals(actual.getAddInputCalls(), 3 * 2);
assertEquals(actual.getAddInputWall(), new Duration(3 * 3, NANOSECONDS));
assertEquals(actual.getAddInputCpu(), new Duration(3 * 4, NANOSECONDS));
assertEquals(actual.getAddInputUser(), new Duration(3 * 5, NANOSECONDS));
assertEquals(actual.getInputDataSize(), new DataSize(3 * 6, BYTE));
assertEquals(actual.getInputPositions(), 3 * 7);
assertEquals(actual.getSumSquaredInputPositions(), 3 * 8.0);
assertEquals(actual.getGetOutputCalls(), 3 * 9);
assertEquals(actual.getGetOutputWall(), new Duration(3 * 10, NANOSECONDS));
assertEquals(actual.getGetOutputCpu(), new Duration(3 * 11, NANOSECONDS));
assertEquals(actual.getGetOutputUser(), new Duration(3 * 12, NANOSECONDS));
assertEquals(actual.getOutputDataSize(), new DataSize(3 * 13, BYTE));
assertEquals(actual.getOutputPositions(), 3 * 14);
assertEquals(actual.getBlockedWall(), new Duration(3 * 15, NANOSECONDS));
assertEquals(actual.getFinishCalls(), 3 * 16);
assertEquals(actual.getFinishWall(), new Duration(3 * 17, NANOSECONDS));
assertEquals(actual.getFinishCpu(), new Duration(3 * 18, NANOSECONDS));
assertEquals(actual.getFinishUser(), new Duration(3 * 19, NANOSECONDS));
assertEquals(actual.getMemoryReservation(), new DataSize(3 * 20, BYTE));
assertEquals(actual.getSystemMemoryReservation(), new DataSize(3 * 21, BYTE));
assertEquals(actual.getInfo().getClass(), PartitionedOutputInfo.class);
assertEquals(((PartitionedOutputInfo) actual.getInfo()).getPagesAdded(), 3 * MERGEABLE_INFO.getPagesAdded());
}
Aggregations