Examples with DataSize io.airlift.units.DataSize used on opensource projects

Example 76 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestTopNOperator method testReverseOrder.

@Test
public void testReverseOrder() throws Exception {
    List<Page> input = rowPagesBuilder(BIGINT, DOUBLE).row(1L, 0.1).row(2L, 0.2).pageBreak().row(-1L, -0.1).row(4L, 0.4).pageBreak().row(5L, 0.5).row(4L, 0.41).row(6L, 0.6).pageBreak().build();
    TopNOperatorFactory operatorFactory = new TopNOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT, DOUBLE), 2, ImmutableList.of(0), ImmutableList.of(ASC_NULLS_LAST), false, new DataSize(16, MEGABYTE));
    MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT, DOUBLE).row(-1L, -0.1).row(1L, 0.1).build();
    assertOperatorEquals(operatorFactory, driverContext, input, expected);
}

Example 77 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class OrcTester method createCustomOrcRecordReader.

static OrcRecordReader createCustomOrcRecordReader(TempFile tempFile, MetadataReader metadataReader, OrcPredicate predicate, Type type) throws IOException {
    OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE));
    OrcReader orcReader = new OrcReader(orcDataSource, metadataReader, new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE));
    assertEquals(orcReader.getColumnNames(), ImmutableList.of("test"));
    assertEquals(orcReader.getFooter().getRowsInRowGroup(), 10_000);
    return orcReader.createRecordReader(ImmutableMap.of(0, type), predicate, HIVE_STORAGE_TIME_ZONE, new AggregatedMemoryContext());
}

Example 78 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestCachingOrcDataSource method testTinyStripesReadCacheAt.

@Test
public void testTinyStripesReadCacheAt() throws IOException {
    DataSize maxMergeDistance = new DataSize(1, Unit.MEGABYTE);
    DataSize maxReadSize = new DataSize(8, Unit.MEGABYTE);
    TestingOrcDataSource testingOrcDataSource = new TestingOrcDataSource(FakeOrcDataSource.INSTANCE);
    CachingOrcDataSource cachingOrcDataSource = new CachingOrcDataSource(testingOrcDataSource, createTinyStripesRangeFinder(ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10), new StripeInformation(123, 33, 10, 10, 10), new StripeInformation(123, 63, 1048576 * 8 - 20, 10, 10)), maxMergeDistance, maxReadSize));
    cachingOrcDataSource.readCacheAt(3);
    assertEquals(testingOrcDataSource.getLastReadRanges(), ImmutableList.of(new DiskRange(3, 60)));
    cachingOrcDataSource.readCacheAt(63);
    assertEquals(testingOrcDataSource.getLastReadRanges(), ImmutableList.of(new DiskRange(63, 8 * 1048576)));
    testingOrcDataSource = new TestingOrcDataSource(FakeOrcDataSource.INSTANCE);
    cachingOrcDataSource = new CachingOrcDataSource(testingOrcDataSource, createTinyStripesRangeFinder(ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10), new StripeInformation(123, 33, 10, 10, 10), new StripeInformation(123, 63, 1048576 * 8 - 20, 10, 10)), maxMergeDistance, maxReadSize));
    // read at the end of a stripe
    cachingOrcDataSource.readCacheAt(62);
    assertEquals(testingOrcDataSource.getLastReadRanges(), ImmutableList.of(new DiskRange(3, 60)));
    cachingOrcDataSource.readCacheAt(63);
    assertEquals(testingOrcDataSource.getLastReadRanges(), ImmutableList.of(new DiskRange(63, 8 * 1048576)));
    testingOrcDataSource = new TestingOrcDataSource(FakeOrcDataSource.INSTANCE);
    cachingOrcDataSource = new CachingOrcDataSource(testingOrcDataSource, createTinyStripesRangeFinder(ImmutableList.of(new StripeInformation(123, 3, 1, 0, 0), new StripeInformation(123, 4, 1048576, 1048576, 1048576 * 3), new StripeInformation(123, 4 + 1048576 * 5, 1048576, 1048576, 1048576)), maxMergeDistance, maxReadSize));
    cachingOrcDataSource.readCacheAt(3);
    assertEquals(testingOrcDataSource.getLastReadRanges(), ImmutableList.of(new DiskRange(3, 1 + 1048576 * 5)));
    cachingOrcDataSource.readCacheAt(4 + 1048576 * 5);
    assertEquals(testingOrcDataSource.getLastReadRanges(), ImmutableList.of(new DiskRange(4 + 1048576 * 5, 3 * 1048576)));
}

Example 79 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestCachingOrcDataSource method testWrapWithCacheIfTinyStripes.

@Test
public void testWrapWithCacheIfTinyStripes() throws IOException {
    DataSize maxMergeDistance = new DataSize(1, Unit.MEGABYTE);
    DataSize maxReadSize = new DataSize(8, Unit.MEGABYTE);
    OrcDataSource actual = wrapWithCacheIfTinyStripes(FakeOrcDataSource.INSTANCE, ImmutableList.of(), maxMergeDistance, maxReadSize);
    assertInstanceOf(actual, CachingOrcDataSource.class);
    actual = wrapWithCacheIfTinyStripes(FakeOrcDataSource.INSTANCE, ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10)), maxMergeDistance, maxReadSize);
    assertInstanceOf(actual, CachingOrcDataSource.class);
    actual = wrapWithCacheIfTinyStripes(FakeOrcDataSource.INSTANCE, ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10), new StripeInformation(123, 33, 10, 10, 10), new StripeInformation(123, 63, 10, 10, 10)), maxMergeDistance, maxReadSize);
    assertInstanceOf(actual, CachingOrcDataSource.class);
    actual = wrapWithCacheIfTinyStripes(FakeOrcDataSource.INSTANCE, ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10), new StripeInformation(123, 33, 10, 10, 10), new StripeInformation(123, 63, 1048576 * 8 - 20, 10, 10)), maxMergeDistance, maxReadSize);
    assertInstanceOf(actual, CachingOrcDataSource.class);
    actual = wrapWithCacheIfTinyStripes(FakeOrcDataSource.INSTANCE, ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10), new StripeInformation(123, 33, 10, 10, 10), new StripeInformation(123, 63, 1048576 * 8 - 20 + 1, 10, 10)), maxMergeDistance, maxReadSize);
    assertNotInstanceOf(actual, CachingOrcDataSource.class);
}

Example 80 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestOrcReaderPositions method testReadUserMetadata.

@Test
public void testReadUserMetadata() throws Exception {
    try (TempFile tempFile = new TempFile()) {
        Map<String, String> metadata = ImmutableMap.of("a", "ala", "b", "ma", "c", "kota");
        createFileWithOnlyUserMetadata(tempFile.getFile(), metadata);
        OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), new DataSize(1, DataSize.Unit.MEGABYTE), new DataSize(1, DataSize.Unit.MEGABYTE), new DataSize(1, DataSize.Unit.MEGABYTE));
        OrcReader orcReader = new OrcReader(orcDataSource, new OrcMetadataReader(), new DataSize(1, DataSize.Unit.MEGABYTE), new DataSize(1, DataSize.Unit.MEGABYTE));
        Footer footer = orcReader.getFooter();
        Map<String, String> readMetadata = Maps.transformValues(footer.getUserMetadata(), Slice::toStringAscii);
        assertEquals(readMetadata, metadata);
    }
}