Examples with BlockEncodingManager com.facebook.presto.common.block.BlockEncodingManager used on opensource projects

Example 1 with BlockEncodingManager

use of com.facebook.presto.common.block.BlockEncodingManager in project presto by prestodb.

the class HiveTestUtils method getDefaultHiveBatchPageSourceFactories.

public static Set<HiveBatchPageSourceFactory> getDefaultHiveBatchPageSourceFactories(HiveClientConfig hiveClientConfig, MetastoreClientConfig metastoreClientConfig) {
    FileFormatDataSourceStats stats = new FileFormatDataSourceStats();
    HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig, metastoreClientConfig);
    return ImmutableSet.<HiveBatchPageSourceFactory>builder().add(new RcFilePageSourceFactory(FUNCTION_AND_TYPE_MANAGER, testHdfsEnvironment, stats)).add(new OrcBatchPageSourceFactory(FUNCTION_AND_TYPE_MANAGER, FUNCTION_RESOLUTION, hiveClientConfig, testHdfsEnvironment, stats, new StorageOrcFileTailSource(), StripeMetadataSourceFactory.of(new StorageStripeMetadataSource()))).add(new DwrfBatchPageSourceFactory(FUNCTION_AND_TYPE_MANAGER, FUNCTION_RESOLUTION, hiveClientConfig, testHdfsEnvironment, stats, new StorageOrcFileTailSource(), StripeMetadataSourceFactory.of(new StorageStripeMetadataSource()), NO_ENCRYPTION)).add(new ParquetPageSourceFactory(FUNCTION_AND_TYPE_MANAGER, FUNCTION_RESOLUTION, testHdfsEnvironment, stats, new MetadataReader())).add(new PageFilePageSourceFactory(testHdfsEnvironment, new BlockEncodingManager())).build();
}

Example 2 with BlockEncodingManager

use of com.facebook.presto.common.block.BlockEncodingManager in project presto by prestodb.

the class TestTempStorageSingleStreamSpiller method assertSpill.

private void assertSpill(boolean compression, boolean encryption) throws Exception {
    File spillPath = new File(tempDirectory, UUID.randomUUID().toString());
    TempStorageSingleStreamSpillerFactory spillerFactory = new TempStorageSingleStreamSpillerFactory(new TestingTempStorageManager(spillPath.toString()), // executor won't be closed, because we don't call destroy() on the spiller factory
    executor, new BlockEncodingManager(), new SpillerStats(), compression, encryption, LocalTempStorage.NAME);
    LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext("test");
    SingleStreamSpiller singleStreamSpiller = spillerFactory.create(TYPES, new TestingSpillContext(), memoryContext);
    assertTrue(singleStreamSpiller instanceof TempStorageSingleStreamSpiller);
    TempStorageSingleStreamSpiller spiller = (TempStorageSingleStreamSpiller) singleStreamSpiller;
    Page page = buildPage();
    // The spillers will reserve memory in their constructors
    int retainedSizeForEmptyDataSink = toIntExact(new OutputStreamDataSink(new DynamicSliceOutput(0)).getRetainedSizeInBytes());
    assertEquals(memoryContext.getBytes(), retainedSizeForEmptyDataSink);
    spiller.spill(page).get();
    spiller.spill(Iterators.forArray(page, page, page)).get();
    assertEquals(listFiles(spillPath.toPath()).size(), 1);
    // The spillers release their memory reservations when they are closed, therefore at this point
    // they will have non-zero memory reservation.
    // assertEquals(memoryContext.getBytes(), 0);
    Iterator<Page> spilledPagesIterator = spiller.getSpilledPages();
    assertEquals(memoryContext.getBytes(), retainedSizeForEmptyDataSink);
    ImmutableList<Page> spilledPages = ImmutableList.copyOf(spilledPagesIterator);
    // The spillers release their memory reservations when they are closed, therefore at this point
    // they will have non-zero memory reservation.
    // assertEquals(memoryContext.getBytes(), 0);
    assertEquals(4, spilledPages.size());
    for (int i = 0; i < 4; ++i) {
        PageAssertions.assertPageEquals(TYPES, page, spilledPages.get(i));
    }
    // Assert the spill codec flags match the expected configuration
    try (InputStream is = newInputStream(listFiles(spillPath.toPath()).get(0))) {
        Iterator<SerializedPage> serializedPages = PagesSerdeUtil.readSerializedPages(new InputStreamSliceInput(is));
        assertTrue(serializedPages.hasNext(), "at least one page should be successfully read back");
        byte markers = serializedPages.next().getPageCodecMarkers();
        assertEquals(PageCodecMarker.COMPRESSED.isSet(markers), compression);
        assertEquals(PageCodecMarker.ENCRYPTED.isSet(markers), encryption);
    }
    spiller.close();
    assertEquals(listFiles(spillPath.toPath()).size(), 0);
    assertEquals(memoryContext.getBytes(), 0);
}

Example 3 with BlockEncodingManager

use of com.facebook.presto.common.block.BlockEncodingManager in project presto by prestodb.

the class TestBinaryFileSpiller method setUp.

@BeforeMethod
public void setUp() {
    blockEncodingSerde = new BlockEncodingManager();
    spillerStats = new SpillerStats();
    FeaturesConfig featuresConfig = new FeaturesConfig();
    featuresConfig.setSpillerSpillPaths(spillPath.getAbsolutePath());
    featuresConfig.setSpillMaxUsedSpaceThreshold(1.0);
    NodeSpillConfig nodeSpillConfig = new NodeSpillConfig();
    singleStreamSpillerFactory = new FileSingleStreamSpillerFactory(blockEncodingSerde, spillerStats, featuresConfig, nodeSpillConfig);
    factory = new GenericSpillerFactory(singleStreamSpillerFactory);
    PagesSerdeFactory pagesSerdeFactory = new PagesSerdeFactory(requireNonNull(blockEncodingSerde, "blockEncodingSerde is null"), nodeSpillConfig.isSpillCompressionEnabled());
    pagesSerde = pagesSerdeFactory.createPagesSerde();
    memoryContext = newSimpleAggregatedMemoryContext();
}

Example 4 with BlockEncodingManager

use of com.facebook.presto.common.block.BlockEncodingManager in project presto by prestodb.

the class TestFileSingleStreamSpillerFactory method testDistributesSpillOverPaths.

@Test
public void testDistributesSpillOverPaths() throws Exception {
    List<Type> types = ImmutableList.of(BIGINT);
    BlockEncodingSerde blockEncodingSerde = new BlockEncodingManager();
    List<Path> spillPaths = ImmutableList.of(spillPath1.toPath(), spillPath2.toPath());
    FileSingleStreamSpillerFactory spillerFactory = new FileSingleStreamSpillerFactory(// executor won't be closed, because we don't call destroy() on the spiller factory
    executor, blockEncodingSerde, new SpillerStats(), spillPaths, 1.0, false, false);
    assertEquals(listFiles(spillPath1.toPath()).size(), 0);
    assertEquals(listFiles(spillPath2.toPath()).size(), 0);
    Page page = buildPage();
    List<SingleStreamSpiller> spillers = new ArrayList<>();
    for (int i = 0; i < 10; ++i) {
        SingleStreamSpiller singleStreamSpiller = spillerFactory.create(types, new TestingSpillContext(), newSimpleAggregatedMemoryContext().newLocalMemoryContext("test"));
        getUnchecked(singleStreamSpiller.spill(page));
        spillers.add(singleStreamSpiller);
    }
    assertEquals(listFiles(spillPath1.toPath()).size(), 5);
    assertEquals(listFiles(spillPath2.toPath()).size(), 5);
    spillers.forEach(SingleStreamSpiller::close);
    assertEquals(listFiles(spillPath1.toPath()).size(), 0);
    assertEquals(listFiles(spillPath2.toPath()).size(), 0);
}

Example 5 with BlockEncodingManager

use of com.facebook.presto.common.block.BlockEncodingManager in project presto by prestodb.

the class TestFileSingleStreamSpillerFactory method throwIfNoSpillPaths.

@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "No spill paths configured")
public void throwIfNoSpillPaths() {
    List<Path> spillPaths = emptyList();
    List<Type> types = ImmutableList.of(BIGINT);
    FileSingleStreamSpillerFactory spillerFactory = new FileSingleStreamSpillerFactory(// executor won't be closed, because we don't call destroy() on the spiller factory
    executor, new BlockEncodingManager(), new SpillerStats(), spillPaths, 1.0, false, false);
    spillerFactory.create(types, new TestingSpillContext(), newSimpleAggregatedMemoryContext().newLocalMemoryContext("test"));
}