Examples with DataSize io.airlift.units.DataSize used on opensource projects

Example 66 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestHashAggregationOperator method testHashBuilderResizeLimit.

@Test(dataProvider = "hashEnabled", expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded local memory limit of 3MB")
public void testHashBuilderResizeLimit(boolean hashEnabled) {
    BlockBuilder builder = VARCHAR.createBlockBuilder(new BlockBuilderStatus(), 1, DEFAULT_MAX_BLOCK_SIZE_IN_BYTES);
    // this must be larger than DEFAULT_MAX_BLOCK_SIZE, 64K
    VARCHAR.writeSlice(builder, Slices.allocate(5_000_000));
    builder.build();
    List<Integer> hashChannels = Ints.asList(0);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, hashChannels, VARCHAR);
    List<Page> input = rowPagesBuilder.addSequencePage(10, 100).addBlocksPage(builder.build()).addSequencePage(10, 100).build();
    DriverContext driverContext = createTaskContext(executor, TEST_SESSION, new DataSize(3, MEGABYTE)).addPipelineContext(0, true, true).addDriverContext();
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(VARCHAR), hashChannels, ImmutableList.of(), Step.SINGLE, ImmutableList.of(COUNT.bind(ImmutableList.of(0), Optional.empty())), rowPagesBuilder.getHashChannel(), Optional.empty(), 100_000, new DataSize(16, MEGABYTE), joinCompiler);
    toPages(operatorFactory, driverContext, input);
}

Example 67 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestWindowOperator method testMemoryLimit.

@Test(expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded local memory limit of 10B")
public void testMemoryLimit() throws Exception {
    List<Page> input = rowPagesBuilder(BIGINT, DOUBLE).row(1L, 0.1).row(2L, 0.2).pageBreak().row(-1L, -0.1).row(4L, 0.4).build();
    DriverContext driverContext = createTaskContext(executor, TEST_SESSION, new DataSize(10, Unit.BYTE)).addPipelineContext(0, true, true).addDriverContext();
    WindowOperatorFactory operatorFactory = createFactoryUnbounded(ImmutableList.of(BIGINT, DOUBLE), Ints.asList(1), ROW_NUMBER, Ints.asList(), Ints.asList(0), ImmutableList.copyOf(new SortOrder[] { SortOrder.ASC_NULLS_LAST }));
    toPages(operatorFactory, driverContext, input);
}

Example 68 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestDriverStats method assertExpectedDriverStats.

public static void assertExpectedDriverStats(DriverStats actual) {
    assertEquals(actual.getCreateTime(), new DateTime(1, UTC));
    assertEquals(actual.getStartTime(), new DateTime(2, UTC));
    assertEquals(actual.getEndTime(), new DateTime(3, UTC));
    assertEquals(actual.getQueuedTime(), new Duration(4, NANOSECONDS));
    assertEquals(actual.getElapsedTime(), new Duration(5, NANOSECONDS));
    assertEquals(actual.getMemoryReservation(), new DataSize(6, BYTE));
    assertEquals(actual.getPeakMemoryReservation(), new DataSize(19, BYTE));
    assertEquals(actual.getSystemMemoryReservation(), new DataSize(7, BYTE));
    assertEquals(actual.getTotalScheduledTime(), new Duration(8, NANOSECONDS));
    assertEquals(actual.getTotalCpuTime(), new Duration(9, NANOSECONDS));
    assertEquals(actual.getTotalUserTime(), new Duration(10, NANOSECONDS));
    assertEquals(actual.getTotalBlockedTime(), new Duration(11, NANOSECONDS));
    assertEquals(actual.getRawInputDataSize(), new DataSize(12, BYTE));
    assertEquals(actual.getRawInputPositions(), 13);
    assertEquals(actual.getRawInputReadTime(), new Duration(14, NANOSECONDS));
    assertEquals(actual.getProcessedInputDataSize(), new DataSize(15, BYTE));
    assertEquals(actual.getProcessedInputPositions(), 16);
    assertEquals(actual.getOutputDataSize(), new DataSize(17, BYTE));
    assertEquals(actual.getOutputPositions(), 18);
    assertEquals(actual.getOperatorStats().size(), 1);
    assertExpectedOperatorStats(actual.getOperatorStats().get(0));
}

Example 69 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestExchangeClient method testBufferLimit.

@Test
public void testBufferLimit() throws Exception {
    DataSize maxResponseSize = new DataSize(1, Unit.BYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);
    URI location = URI.create("http://localhost:8080");
    // add a pages
    processor.addPage(location, createPage(1));
    processor.addPage(location, createPage(2));
    processor.addPage(location, createPage(3));
    processor.setComplete(location);
    @SuppressWarnings("resource") ExchangeClient exchangeClient = new ExchangeClient(new DataSize(1, Unit.BYTE), maxResponseSize, 1, new Duration(1, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), new TestingHttpClient(processor, newCachedThreadPool(daemonThreadsNamed("test-%s"))), executor, deltaMemoryInBytes -> {
    });
    exchangeClient.addLocation(location);
    exchangeClient.noMoreLocations();
    assertEquals(exchangeClient.isClosed(), false);
    long start = System.nanoTime();
    // start fetching pages
    exchangeClient.scheduleRequestIfNecessary();
    // wait for a page to be fetched
    do {
        // there is no thread coordination here, so sleep is the best we can do
        assertLessThan(Duration.nanosSince(start), new Duration(5, TimeUnit.SECONDS));
        sleepUninterruptibly(100, MILLISECONDS);
    } while (exchangeClient.getStatus().getBufferedPages() == 0);
    // client should have sent a single request for a single page
    assertEquals(exchangeClient.getStatus().getBufferedPages(), 1);
    assertTrue(exchangeClient.getStatus().getBufferedBytes() > 0);
    assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "queued", 1, 1, 1, "not scheduled");
    // remove the page and wait for the client to fetch another page
    assertPageEquals(exchangeClient.getNextPage(new Duration(0, TimeUnit.SECONDS)), createPage(1));
    do {
        assertLessThan(Duration.nanosSince(start), new Duration(5, TimeUnit.SECONDS));
        sleepUninterruptibly(100, MILLISECONDS);
    } while (exchangeClient.getStatus().getBufferedPages() == 0);
    // client should have sent a single request for a single page
    assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "queued", 2, 2, 2, "not scheduled");
    assertEquals(exchangeClient.getStatus().getBufferedPages(), 1);
    assertTrue(exchangeClient.getStatus().getBufferedBytes() > 0);
    // remove the page and wait for the client to fetch another page
    assertPageEquals(exchangeClient.getNextPage(new Duration(0, TimeUnit.SECONDS)), createPage(2));
    do {
        assertLessThan(Duration.nanosSince(start), new Duration(5, TimeUnit.SECONDS));
        sleepUninterruptibly(100, MILLISECONDS);
    } while (exchangeClient.getStatus().getBufferedPages() == 0);
    // client should have sent a single request for a single page
    assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "queued", 3, 3, 3, "not scheduled");
    assertEquals(exchangeClient.getStatus().getBufferedPages(), 1);
    assertTrue(exchangeClient.getStatus().getBufferedBytes() > 0);
    // remove last page
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(3));
    //  wait for client to decide there are no more pages
    assertNull(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)));
    assertEquals(exchangeClient.getStatus().getBufferedPages(), 0);
    assertTrue(exchangeClient.getStatus().getBufferedBytes() == 0);
    assertEquals(exchangeClient.isClosed(), true);
    assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "closed", 3, 5, 5, "not scheduled");
}

Example 70 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestExchangeClient method testAddLocation.

@Test(timeOut = 10000)
public void testAddLocation() throws Exception {
    DataSize maxResponseSize = new DataSize(10, Unit.MEGABYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);
    @SuppressWarnings("resource") ExchangeClient exchangeClient = new ExchangeClient(new DataSize(32, Unit.MEGABYTE), maxResponseSize, 1, new Duration(1, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), new TestingHttpClient(processor, newCachedThreadPool(daemonThreadsNamed("test-%s"))), executor, deltaMemoryInBytes -> {
    });
    URI location1 = URI.create("http://localhost:8081/foo");
    processor.addPage(location1, createPage(1));
    processor.addPage(location1, createPage(2));
    processor.addPage(location1, createPage(3));
    processor.setComplete(location1);
    exchangeClient.addLocation(location1);
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(1));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(2));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(3));
    assertNull(exchangeClient.getNextPage(new Duration(10, MILLISECONDS)));
    assertEquals(exchangeClient.isClosed(), false);
    URI location2 = URI.create("http://localhost:8082/bar");
    processor.addPage(location2, createPage(4));
    processor.addPage(location2, createPage(5));
    processor.addPage(location2, createPage(6));
    processor.setComplete(location2);
    exchangeClient.addLocation(location2);
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(4));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(5));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(6));
    assertNull(exchangeClient.getNextPage(new Duration(10, MILLISECONDS)));
    assertEquals(exchangeClient.isClosed(), false);
    exchangeClient.noMoreLocations();
    // receive a final GONE response, so just spin until it's closed or the test times out.
    while (!exchangeClient.isClosed()) {
        Thread.sleep(1);
    }
    ImmutableMap<URI, PageBufferClientStatus> statuses = uniqueIndex(exchangeClient.getStatus().getPageBufferClientStatuses(), PageBufferClientStatus::getUri);
    assertStatus(statuses.get(location1), location1, "closed", 3, 3, 3, "not scheduled");
    assertStatus(statuses.get(location2), location2, "closed", 3, 3, 3, "not scheduled");
}