Examples with ColumnInfo com.facebook.presto.raptor.metadata.ColumnInfo used on opensource projects

Example 6 with ColumnInfo

use of com.facebook.presto.raptor.metadata.ColumnInfo in project presto by prestodb.

the class TestShardEjector method testEjector.

@Test(invocationCount = 20)
public void testEjector() throws Exception {
    NodeManager nodeManager = createNodeManager("node1", "node2", "node3", "node4", "node5");
    ShardEjector ejector = new ShardEjector(nodeManager.getCurrentNode().getNodeIdentifier(), nodeManager::getWorkerNodes, shardManager, storageService, new Duration(1, HOURS), Optional.of(new TestingBackupStore()), "test");
    List<ShardInfo> shards = ImmutableList.<ShardInfo>builder().add(shardInfo("node1", 14)).add(shardInfo("node1", 13)).add(shardInfo("node1", 12)).add(shardInfo("node1", 11)).add(shardInfo("node1", 10)).add(shardInfo("node1", 10)).add(shardInfo("node1", 10)).add(shardInfo("node1", 10)).add(shardInfo("node2", 5)).add(shardInfo("node2", 5)).add(shardInfo("node3", 10)).add(shardInfo("node4", 10)).add(shardInfo("node5", 10)).add(shardInfo("node6", 200)).build();
    long tableId = createTable("test");
    List<ColumnInfo> columns = ImmutableList.of(new ColumnInfo(1, BIGINT));
    shardManager.createTable(tableId, columns, false, OptionalLong.empty());
    long transactionId = shardManager.beginTransaction();
    shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0);
    for (ShardInfo shard : shards.subList(0, 8)) {
        File file = storageService.getStorageFile(shard.getShardUuid());
        storageService.createParents(file);
        assertTrue(file.createNewFile());
    }
    ejector.process();
    shardManager.getShardNodes(tableId, TupleDomain.all());
    Set<UUID> ejectedShards = shards.subList(0, 4).stream().map(ShardInfo::getShardUuid).collect(toSet());
    Set<UUID> keptShards = shards.subList(4, 8).stream().map(ShardInfo::getShardUuid).collect(toSet());
    Set<UUID> remaining = uuids(shardManager.getNodeShards("node1"));
    for (UUID uuid : ejectedShards) {
        assertFalse(remaining.contains(uuid));
        assertFalse(storageService.getStorageFile(uuid).exists());
    }
    assertEquals(remaining, keptShards);
    for (UUID uuid : keptShards) {
        assertTrue(storageService.getStorageFile(uuid).exists());
    }
    Set<UUID> others = ImmutableSet.<UUID>builder().addAll(uuids(shardManager.getNodeShards("node2"))).addAll(uuids(shardManager.getNodeShards("node3"))).addAll(uuids(shardManager.getNodeShards("node4"))).addAll(uuids(shardManager.getNodeShards("node5"))).build();
    assertTrue(others.containsAll(ejectedShards));
}

Example 7 with ColumnInfo

use of com.facebook.presto.raptor.metadata.ColumnInfo in project presto by prestodb.

the class OrcStorageManager method computeShardStats.

private List<ColumnStats> computeShardStats(File file) {
    try (OrcDataSource dataSource = fileOrcDataSource(defaultReaderAttributes, file)) {
        OrcReader reader = new OrcReader(dataSource, new OrcMetadataReader(), defaultReaderAttributes.getMaxMergeDistance(), defaultReaderAttributes.getMaxReadSize());
        ImmutableList.Builder<ColumnStats> list = ImmutableList.builder();
        for (ColumnInfo info : getColumnInfo(reader)) {
            computeColumnStats(reader, info.getColumnId(), info.getType()).ifPresent(list::add);
        }
        return list.build();
    } catch (IOException e) {
        throw new PrestoException(RAPTOR_ERROR, "Failed to read file: " + file, e);
    }
}

Example 8 with ColumnInfo

use of com.facebook.presto.raptor.metadata.ColumnInfo in project presto by prestodb.

the class TestBucketBalancer method createBucketedTable.

private long createBucketedTable(String tableName, long distributionId, DataSize compressedSize) {
    MetadataDao dao = dbi.onDemand(MetadataDao.class);
    long tableId = dao.insertTable("test", tableName, false, false, distributionId, 0);
    List<ColumnInfo> columnsA = ImmutableList.of(new ColumnInfo(1, BIGINT));
    shardManager.createTable(tableId, columnsA, false, OptionalLong.empty());
    metadataDao.updateTableStats(tableId, 1024, 1024 * 1024 * 1024, compressedSize.toBytes(), compressedSize.toBytes() * 2);
    return tableId;
}