Examples with StorageManager com.facebook.presto.raptor.storage.StorageManager used on opensource projects

Example 1 with StorageManager

use of com.facebook.presto.raptor.storage.StorageManager in project presto by prestodb.

the class TestShardCompactor method testShardCompactorSorted.

@Test(dataProvider = "useOptimizedOrcWriter")
public void testShardCompactorSorted(boolean useOptimizedOrcWriter) throws Exception {
    StorageManager storageManager = createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS);
    List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), DATE, TIMESTAMP, DOUBLE);
    List<Long> columnIds = ImmutableList.of(3L, 7L, 2L, 1L, 5L);
    List<Long> sortColumnIds = ImmutableList.of(1L, 2L, 3L, 5L, 7L);
    List<SortOrder> sortOrders = nCopies(sortColumnIds.size(), ASC_NULLS_FIRST);
    List<Integer> sortIndexes = sortColumnIds.stream().map(columnIds::indexOf).collect(toList());
    List<ShardInfo> inputShards = createSortedShards(storageManager, columnIds, columnTypes, sortIndexes, sortOrders, 2);
    assertEquals(inputShards.size(), 2);
    long totalRows = inputShards.stream().mapToLong(ShardInfo::getRowCount).sum();
    long expectedOutputShards = computeExpectedOutputShards(totalRows);
    Map<UUID, Optional<UUID>> inputUuids = new HashMap<>();
    for (ShardInfo shardInfo : inputShards) {
        inputUuids.put(shardInfo.getShardUuid(), Optional.empty());
    }
    long transactionId = 1;
    ShardCompactor compactor = new ShardCompactor(storageManager, READER_ATTRIBUTES);
    List<ShardInfo> outputShards = compactor.compactSorted(transactionId, false, OptionalInt.empty(), inputUuids, getColumnInfo(columnIds, columnTypes), sortColumnIds, sortOrders);
    List<UUID> outputUuids = outputShards.stream().map(ShardInfo::getShardUuid).collect(toList());
    assertEquals(outputShards.size(), expectedOutputShards);
    assertShardEqualsSorted(storageManager, inputUuids.keySet(), outputUuids, columnIds, columnTypes, sortIndexes, sortOrders);
}

Example 2 with StorageManager

use of com.facebook.presto.raptor.storage.StorageManager in project presto by prestodb.

the class TestShardCompactor method testShardCompactor.

@Test(dataProvider = "useOptimizedOrcWriter")
public void testShardCompactor(boolean useOptimizedOrcWriter) throws Exception {
    StorageManager storageManager = createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS);
    List<Long> columnIds = ImmutableList.of(3L, 7L, 2L, 1L, 5L);
    List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), DOUBLE, DATE, TIMESTAMP);
    List<ShardInfo> inputShards = createShards(storageManager, columnIds, columnTypes, 3);
    assertEquals(inputShards.size(), 3);
    long totalRows = inputShards.stream().mapToLong(ShardInfo::getRowCount).sum();
    long expectedOutputShards = computeExpectedOutputShards(totalRows);
    Map<UUID, Optional<UUID>> inputUuids = new HashMap<>();
    for (ShardInfo shardInfo : inputShards) {
        inputUuids.put(shardInfo.getShardUuid(), Optional.empty());
    }
    long transactionId = 1;
    ShardCompactor compactor = new ShardCompactor(storageManager, READER_ATTRIBUTES);
    List<ShardInfo> outputShards = compactor.compact(transactionId, false, OptionalInt.empty(), inputUuids, getColumnInfo(columnIds, columnTypes));
    assertEquals(outputShards.size(), expectedOutputShards);
    Set<UUID> outputUuids = outputShards.stream().map(ShardInfo::getShardUuid).collect(toSet());
    assertShardEqualsIgnoreOrder(storageManager, inputUuids.keySet(), outputUuids, columnIds, columnTypes);
}

Example 3 with StorageManager

use of com.facebook.presto.raptor.storage.StorageManager in project presto by prestodb.

the class TestShardCompactor method testShardCompactorWithDelta.

@Test
public void testShardCompactorWithDelta() throws Exception {
    StorageManager storageManager = createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS);
    List<Long> columnIds = ImmutableList.of(3L, 7L, 2L, 1L, 5L);
    List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), DOUBLE, DATE, TIMESTAMP);
    List<ShardInfo> inputShards = createShards(storageManager, columnIds, columnTypes, 3);
    assertEquals(inputShards.size(), 3);
    List<Long> deltaColumnIds = ImmutableList.of(1L);
    List<Type> deltaColumnTypes = ImmutableList.of(BIGINT);
    StoragePageSink deltaSink = createStoragePageSink(storageManager, deltaColumnIds, deltaColumnTypes);
    List<Page> deltaPages = rowPagesBuilder(deltaColumnTypes).row(1L).row(2L).build();
    deltaSink.appendPages(deltaPages);
    List<ShardInfo> deltaShards = getFutureValue(deltaSink.commit());
    long totalRows = inputShards.stream().mapToLong(ShardInfo::getRowCount).sum();
    long expectedOutputShardsCount = computeExpectedOutputShards(totalRows - 2);
    Map<UUID, Optional<UUID>> inputUuidsMap = new HashMap<>();
    inputUuidsMap.put(inputShards.get(0).getShardUuid(), Optional.of(deltaShards.get(0).getShardUuid()));
    inputUuidsMap.put(inputShards.get(1).getShardUuid(), Optional.empty());
    inputUuidsMap.put(inputShards.get(2).getShardUuid(), Optional.empty());
    long transactionId = 1;
    ShardCompactor compactor = new ShardCompactor(storageManager, READER_ATTRIBUTES);
    List<ShardInfo> outputShards = compactor.compact(transactionId, true, OptionalInt.empty(), inputUuidsMap, getColumnInfo(columnIds, columnTypes));
    assertEquals(outputShards.size(), expectedOutputShardsCount);
    Set<UUID> outputUuids = outputShards.stream().map(ShardInfo::getShardUuid).collect(toSet());
    assertShardEqualsIgnoreOrder(storageManager, inputUuidsMap, outputUuids, columnIds, columnTypes);
}

Example 4 with StorageManager

use of com.facebook.presto.raptor.storage.StorageManager in project presto by prestodb.

the class TestRaptorConnector method setup.

@BeforeMethod
public void setup() throws Exception {
    FunctionAndTypeManager functionAndTypeManager = createTestFunctionAndTypeManager();
    DBI dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime() + "_" + ThreadLocalRandom.current().nextInt());
    dbi.registerMapper(new TableColumn.Mapper(functionAndTypeManager));
    dummyHandle = dbi.open();
    metadataDao = dbi.onDemand(MetadataDao.class);
    createTablesWithRetry(dbi);
    dataDir = Files.createTempDir();
    RaptorConnectorId connectorId = new RaptorConnectorId("test");
    NodeManager nodeManager = new TestingNodeManager();
    NodeSupplier nodeSupplier = nodeManager::getWorkerNodes;
    ShardManager shardManager = createShardManager(dbi);
    StorageManager storageManager = createOrcStorageManager(dbi, dataDir);
    StorageManagerConfig config = new StorageManagerConfig();
    connector = new RaptorConnector(new LifeCycleManager(ImmutableList.of(), null), new TestingNodeManager(), new RaptorMetadataFactory(connectorId, dbi, shardManager, functionAndTypeManager), new RaptorSplitManager(connectorId, nodeSupplier, shardManager, false), new RaptorPageSourceProvider(storageManager), new RaptorPageSinkProvider(storageManager, new PagesIndexPageSorter(new PagesIndex.TestingFactory(false)), new TemporalFunction(DateTimeZone.forID("America/Los_Angeles")), config), new RaptorNodePartitioningProvider(nodeSupplier), new RaptorSessionProperties(config), new RaptorTableProperties(functionAndTypeManager), ImmutableSet.of(), new AllowAllAccessControl(), dbi, ImmutableSet.of());
}