Examples with TableColumn com.facebook.presto.raptor.metadata.TableColumn used on opensource projects

Example 6 with TableColumn

use of com.facebook.presto.raptor.metadata.TableColumn in project presto by prestodb.

the class ShardCompactionManager method filterAndCreateCompactionSets.

private Collection<OrganizationSet> filterAndCreateCompactionSets(long tableId, Collection<ShardMetadata> tableShards) {
    Table tableInfo = metadataDao.getTableInformation(tableId);
    OptionalLong temporalColumnId = tableInfo.getTemporalColumnId();
    if (temporalColumnId.isPresent()) {
        TableColumn tableColumn = metadataDao.getTableColumn(tableId, temporalColumnId.getAsLong());
        if (!isValidTemporalColumn(tableId, tableColumn.getDataType())) {
            return ImmutableSet.of();
        }
    }
    Set<ShardMetadata> filteredShards = tableShards.stream().filter(this::needsCompaction).filter(shard -> !organizer.inProgress(shard.getShardUuid())).collect(toSet());
    Collection<ShardIndexInfo> shardIndexInfos = getOrganizationEligibleShards(dbi, metadataDao, tableInfo, filteredShards, false);
    if (tableInfo.getTemporalColumnId().isPresent()) {
        Set<ShardIndexInfo> temporalShards = shardIndexInfos.stream().filter(shard -> shard.getTemporalRange().isPresent()).collect(toSet());
        return compactionSetCreator.createCompactionSets(tableInfo, temporalShards);
    }
    return compactionSetCreator.createCompactionSets(tableInfo, shardIndexInfos);
}

Example 7 with TableColumn

use of com.facebook.presto.raptor.metadata.TableColumn in project presto by prestodb.

the class TestShardOrganizerUtil method testGetOrganizationEligibleShards.

@Test
public void testGetOrganizationEligibleShards() throws Exception {
    int day1 = 1111;
    int day2 = 2222;
    SchemaTableName tableName = new SchemaTableName("default", "test");
    metadata.createTable(SESSION, tableMetadataBuilder(tableName).column("orderkey", BIGINT).column("orderdate", DATE).column("orderstatus", createVarcharType(3)).property("ordering", ImmutableList.of("orderstatus", "orderkey")).property("temporal_column", "orderdate").build());
    Table tableInfo = metadataDao.getTableInformation(tableName.getSchemaName(), tableName.getTableName());
    List<TableColumn> tableColumns = metadataDao.listTableColumns(tableInfo.getTableId());
    Map<String, TableColumn> tableColumnMap = Maps.uniqueIndex(tableColumns, TableColumn::getColumnName);
    long orderDate = tableColumnMap.get("orderdate").getColumnId();
    long orderKey = tableColumnMap.get("orderkey").getColumnId();
    long orderStatus = tableColumnMap.get("orderstatus").getColumnId();
    List<ShardInfo> shards = ImmutableList.<ShardInfo>builder().add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day1 + 10), new ColumnStats(orderKey, 13L, 14L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day2, day2 + 100), new ColumnStats(orderKey, 2L, 20L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day2), new ColumnStats(orderKey, 2L, 11L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day2), new ColumnStats(orderKey, 2L, null), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, null), new ColumnStats(orderKey, 2L, 11L), new ColumnStats(orderStatus, "aaa", "abc")))).build();
    long transactionId = shardManager.beginTransaction();
    shardManager.commitShards(transactionId, tableInfo.getTableId(), COLUMNS, shards, Optional.empty(), 0);
    Set<ShardMetadata> shardMetadatas = shardManager.getNodeShards("node1");
    Long temporalColumnId = metadataDao.getTemporalColumnId(tableInfo.getTableId());
    TableColumn temporalColumn = metadataDao.getTableColumn(tableInfo.getTableId(), temporalColumnId);
    Set<ShardIndexInfo> actual = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, false));
    List<ShardIndexInfo> expected = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.empty());
    assertEquals(actual, expected);
    List<TableColumn> sortColumns = metadataDao.listSortColumns(tableInfo.getTableId());
    Set<ShardIndexInfo> actualSortRange = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, true));
    List<ShardIndexInfo> expectedSortRange = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.of(sortColumns));
    assertEquals(actualSortRange, expectedSortRange);
}

Example 8 with TableColumn

use of com.facebook.presto.raptor.metadata.TableColumn in project presto by prestodb.

the class TestShardOrganizerUtil method getShardIndexInfo.

private static List<ShardIndexInfo> getShardIndexInfo(Table tableInfo, List<ShardInfo> shards, TableColumn temporalColumn, Optional<List<TableColumn>> sortColumns) {
    long tableId = tableInfo.getTableId();
    Type temporalType = temporalColumn.getDataType();
    ImmutableList.Builder<ShardIndexInfo> builder = ImmutableList.builder();
    for (ShardInfo shard : shards) {
        ColumnStats temporalColumnStats = shard.getColumnStats().stream().filter(columnStats -> columnStats.getColumnId() == temporalColumn.getColumnId()).findFirst().get();
        if (temporalColumnStats.getMin() == null || temporalColumnStats.getMax() == null) {
            continue;
        }
        Optional<ShardRange> sortRange = Optional.empty();
        if (sortColumns.isPresent()) {
            Map<Long, ColumnStats> columnIdToStats = Maps.uniqueIndex(shard.getColumnStats(), ColumnStats::getColumnId);
            ImmutableList.Builder<Type> typesBuilder = ImmutableList.builder();
            ImmutableList.Builder<Object> minBuilder = ImmutableList.builder();
            ImmutableList.Builder<Object> maxBuilder = ImmutableList.builder();
            boolean isShardEligible = true;
            for (TableColumn sortColumn : sortColumns.get()) {
                ColumnStats columnStats = columnIdToStats.get(sortColumn.getColumnId());
                typesBuilder.add(sortColumn.getDataType());
                if (columnStats.getMin() == null || columnStats.getMax() == null) {
                    isShardEligible = false;
                    break;
                }
                minBuilder.add(columnStats.getMin());
                maxBuilder.add(columnStats.getMax());
            }
            if (!isShardEligible) {
                continue;
            }
            List<Type> types = typesBuilder.build();
            List<Object> minValues = minBuilder.build();
            List<Object> maxValues = maxBuilder.build();
            sortRange = Optional.of(ShardRange.of(new Tuple(types, minValues), new Tuple(types, maxValues)));
        }
        builder.add(new ShardIndexInfo(tableId, OptionalInt.empty(), shard.getShardUuid(), shard.getRowCount(), shard.getUncompressedSize(), sortRange, Optional.of(ShardRange.of(new Tuple(temporalType, temporalColumnStats.getMin()), new Tuple(temporalType, temporalColumnStats.getMax())))));
    }
    return builder.build();
}

Example 9 with TableColumn

use of com.facebook.presto.raptor.metadata.TableColumn in project presto by prestodb.

the class RaptorMetadata method addColumn.

@Override
public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnMetadata column) {
    RaptorTableHandle table = (RaptorTableHandle) tableHandle;
    // Always add new columns to the end.
    // TODO: This needs to be updated when we support dropping columns.
    List<TableColumn> existingColumns = dao.listTableColumns(table.getSchemaName(), table.getTableName());
    TableColumn lastColumn = existingColumns.get(existingColumns.size() - 1);
    long columnId = lastColumn.getColumnId() + 1;
    int ordinalPosition = existingColumns.size();
    String type = column.getType().getTypeSignature().toString();
    daoTransaction(dbi, MetadataDao.class, dao -> {
        dao.insertColumn(table.getTableId(), columnId, column.getName(), ordinalPosition, type, null, null);
        dao.updateTableVersion(table.getTableId(), session.getStartTime());
    });
    shardManager.addColumn(table.getTableId(), new ColumnInfo(columnId, column.getType()));
}

Example 10 with TableColumn

use of com.facebook.presto.raptor.metadata.TableColumn in project presto by prestodb.

the class ShardOrganizerUtil method getShardRange.

private static Optional<ShardRange> getShardRange(List<TableColumn> columns, ResultSet resultSet) throws SQLException {
    ImmutableList.Builder<Object> minValuesBuilder = ImmutableList.builder();
    ImmutableList.Builder<Object> maxValuesBuilder = ImmutableList.builder();
    ImmutableList.Builder<Type> typeBuilder = ImmutableList.builder();
    for (TableColumn tableColumn : columns) {
        long columnId = tableColumn.getColumnId();
        Type type = tableColumn.getDataType();
        Object min = getValue(resultSet, type, minColumn(columnId));
        Object max = getValue(resultSet, type, maxColumn(columnId));
        if (min == null || max == null) {
            return Optional.empty();
        }
        minValuesBuilder.add(min);
        maxValuesBuilder.add(max);
        typeBuilder.add(type);
    }
    List<Type> types = typeBuilder.build();
    Tuple minTuple = new Tuple(types, minValuesBuilder.build());
    Tuple maxTuple = new Tuple(types, maxValuesBuilder.build());
    return Optional.of(ShardRange.of(minTuple, maxTuple));
}