Examples with DataSize io.airlift.units.DataSize used on opensource projects

Example 56 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestCachingOrcDataSource method testIntegration.

@Test
public void testIntegration() throws IOException {
    // tiny file
    TestingOrcDataSource orcDataSource = new TestingOrcDataSource(new FileOrcDataSource(tempFile.getFile(), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE)));
    doIntegration(orcDataSource, new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE));
    // read entire file at once
    assertEquals(orcDataSource.getReadCount(), 1);
    // tiny stripes
    orcDataSource = new TestingOrcDataSource(new FileOrcDataSource(tempFile.getFile(), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE)));
    doIntegration(orcDataSource, new DataSize(400, Unit.KILOBYTE), new DataSize(400, Unit.KILOBYTE));
    // footer, first few stripes, last few stripes
    assertEquals(orcDataSource.getReadCount(), 3);
}

Example 57 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestOrcDataSourceUtils method testMergeSingle.

@Test
public void testMergeSingle() {
    List<DiskRange> diskRanges = mergeAdjacentDiskRanges(ImmutableList.of(new DiskRange(100, 100)), new DataSize(0, BYTE), new DataSize(0, BYTE));
    assertEquals(diskRanges, ImmutableList.of(new DiskRange(100, 100)));
}

Example 58 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestOrcDataSourceUtils method testMergeAdjacent.

@Test
public void testMergeAdjacent() {
    List<DiskRange> diskRanges = mergeAdjacentDiskRanges(ImmutableList.of(new DiskRange(100, 100), new DiskRange(200, 100), new DiskRange(300, 100)), new DataSize(0, BYTE), new DataSize(1, GIGABYTE));
    assertEquals(diskRanges, ImmutableList.of(new DiskRange(100, 300)));
}

Example 59 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestHashSemiJoinOperator method testMemoryLimit.

@Test(dataProvider = "hashEnabledValues", expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded local memory limit of.*")
public void testMemoryLimit(boolean hashEnabled) throws Exception {
    DriverContext driverContext = createTaskContext(executor, TEST_SESSION, new DataSize(100, BYTE)).addPipelineContext(0, true, true).addDriverContext();
    OperatorContext operatorContext = driverContext.addOperatorContext(0, new PlanNodeId("test"), ValuesOperator.class.getSimpleName());
    List<Type> buildTypes = ImmutableList.of(BIGINT);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), buildTypes);
    Operator buildOperator = new ValuesOperator(operatorContext, buildTypes, rowPagesBuilder.addSequencePage(10000, 20).build());
    SetBuilderOperatorFactory setBuilderOperatorFactory = new SetBuilderOperatorFactory(1, new PlanNodeId("test"), buildOperator.getTypes().get(0), 0, rowPagesBuilder.getHashChannel(), 10, new JoinCompiler());
    Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext);
    Driver driver = new Driver(driverContext, buildOperator, setBuilderOperator);
    while (!driver.isFinished()) {
        driver.process();
    }
}

Example 60 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestHttpPageBufferClient method testInvalidResponses.

@Test
public void testInvalidResponses() throws Exception {
    CyclicBarrier beforeRequest = new CyclicBarrier(1);
    CyclicBarrier afterRequest = new CyclicBarrier(1);
    StaticRequestProcessor processor = new StaticRequestProcessor(beforeRequest, afterRequest);
    CyclicBarrier requestComplete = new CyclicBarrier(2);
    TestingClientCallback callback = new TestingClientCallback(requestComplete);
    URI location = URI.create("http://localhost:8080");
    HttpPageBufferClient client = new HttpPageBufferClient(new TestingHttpClient(processor, executor), new DataSize(10, Unit.MEGABYTE), new Duration(1, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), location, callback, executor);
    assertStatus(client, location, "queued", 0, 0, 0, 0, "not scheduled");
    // send not found response and verify response was ignored
    processor.setResponse(new TestingResponse(HttpStatus.NOT_FOUND, ImmutableListMultimap.of(CONTENT_TYPE, PRESTO_PAGES), new byte[0]));
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 1);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 1);
    assertInstanceOf(callback.getFailure(), PageTransportErrorException.class);
    assertContains(callback.getFailure().getMessage(), "Expected response code to be 200, but was 404 Not Found");
    assertStatus(client, location, "queued", 0, 1, 1, 1, "not scheduled");
    // send invalid content type response and verify response was ignored
    callback.resetStats();
    processor.setResponse(new TestingResponse(HttpStatus.OK, ImmutableListMultimap.of(CONTENT_TYPE, "INVALID_TYPE"), new byte[0]));
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 1);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 1);
    assertInstanceOf(callback.getFailure(), PageTransportErrorException.class);
    assertContains(callback.getFailure().getMessage(), "Expected application/x-presto-pages response from server but got INVALID_TYPE");
    assertStatus(client, location, "queued", 0, 2, 2, 2, "not scheduled");
    // send unexpected content type response and verify response was ignored
    callback.resetStats();
    processor.setResponse(new TestingResponse(HttpStatus.OK, ImmutableListMultimap.of(CONTENT_TYPE, "text/plain"), new byte[0]));
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 1);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 1);
    assertInstanceOf(callback.getFailure(), PageTransportErrorException.class);
    assertContains(callback.getFailure().getMessage(), "Expected application/x-presto-pages response from server but got text/plain");
    assertStatus(client, location, "queued", 0, 3, 3, 3, "not scheduled");
    // close client and verify
    client.close();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertStatus(client, location, "closed", 0, 3, 4, 3, "not scheduled");
}