Examples with ShardMetadata com.facebook.presto.raptor.metadata.ShardMetadata used on opensource projects

Example 1 with ShardMetadata

use of com.facebook.presto.raptor.metadata.ShardMetadata in project presto by prestodb.

the class ShardOrganizerUtil method getOrganizationEligibleShards.

public static Collection<ShardIndexInfo> getOrganizationEligibleShards(IDBI dbi, MetadataDao metadataDao, Table tableInfo, Collection<ShardMetadata> shards, boolean includeSortColumns) {
    Map<Long, ShardMetadata> shardsById = uniqueIndex(shards, ShardMetadata::getShardId);
    long tableId = tableInfo.getTableId();
    ImmutableList.Builder<String> columnsBuilder = ImmutableList.builder();
    columnsBuilder.add("shard_id");
    // include temporal columns if present
    Optional<TableColumn> temporalColumn = Optional.empty();
    if (tableInfo.getTemporalColumnId().isPresent()) {
        long temporalColumnId = tableInfo.getTemporalColumnId().getAsLong();
        temporalColumn = Optional.of(metadataDao.getTableColumn(tableId, temporalColumnId));
        columnsBuilder.add(minColumn(temporalColumnId), maxColumn(temporalColumnId));
    }
    // include sort columns if needed
    Optional<List<TableColumn>> sortColumns = Optional.empty();
    if (includeSortColumns) {
        sortColumns = Optional.of(metadataDao.listSortColumns(tableId));
        for (TableColumn column : sortColumns.get()) {
            columnsBuilder.add(minColumn(column.getColumnId()), maxColumn(column.getColumnId()));
        }
    }
    String columnToSelect = Joiner.on(",\n").join(columnsBuilder.build());
    ImmutableList.Builder<ShardIndexInfo> indexInfoBuilder = ImmutableList.builder();
    try (Connection connection = dbi.open().getConnection()) {
        for (List<ShardMetadata> partitionedShards : partition(shards, 1000)) {
            String shardIds = Joiner.on(",").join(nCopies(partitionedShards.size(), "?"));
            String sql = format("" + "SELECT %s\n" + "FROM %s\n" + "WHERE shard_id IN (%s)", columnToSelect, shardIndexTable(tableId), shardIds);
            try (PreparedStatement statement = connection.prepareStatement(sql)) {
                for (int i = 0; i < partitionedShards.size(); i++) {
                    statement.setLong(i + 1, partitionedShards.get(i).getShardId());
                }
                try (ResultSet resultSet = statement.executeQuery()) {
                    while (resultSet.next()) {
                        long shardId = resultSet.getLong("shard_id");
                        Optional<ShardRange> sortRange = Optional.empty();
                        if (includeSortColumns) {
                            sortRange = getShardRange(sortColumns.get(), resultSet);
                            if (!sortRange.isPresent()) {
                                continue;
                            }
                        }
                        Optional<ShardRange> temporalRange = Optional.empty();
                        if (temporalColumn.isPresent()) {
                            temporalRange = getShardRange(ImmutableList.of(temporalColumn.get()), resultSet);
                            if (!temporalRange.isPresent()) {
                                continue;
                            }
                        }
                        ShardMetadata shardMetadata = shardsById.get(shardId);
                        indexInfoBuilder.add(toShardIndexInfo(shardMetadata, temporalRange, sortRange));
                    }
                }
            }
        }
    } catch (SQLException e) {
        throw Throwables.propagate(e);
    }
    return indexInfoBuilder.build();
}

Example 2 with ShardMetadata

use of com.facebook.presto.raptor.metadata.ShardMetadata in project presto by prestodb.

the class ShardCompactionManager method filterAndCreateCompactionSets.

private Collection<OrganizationSet> filterAndCreateCompactionSets(long tableId, Collection<ShardMetadata> tableShards) {
    Table tableInfo = metadataDao.getTableInformation(tableId);
    OptionalLong temporalColumnId = tableInfo.getTemporalColumnId();
    if (temporalColumnId.isPresent()) {
        TableColumn tableColumn = metadataDao.getTableColumn(tableId, temporalColumnId.getAsLong());
        if (!isValidTemporalColumn(tableId, tableColumn.getDataType())) {
            return ImmutableSet.of();
        }
    }
    Set<ShardMetadata> filteredShards = tableShards.stream().filter(this::needsCompaction).filter(shard -> !organizer.inProgress(shard.getShardUuid())).collect(toSet());
    Collection<ShardIndexInfo> shardIndexInfos = getOrganizationEligibleShards(dbi, metadataDao, tableInfo, filteredShards, false);
    if (tableInfo.getTemporalColumnId().isPresent()) {
        Set<ShardIndexInfo> temporalShards = shardIndexInfos.stream().filter(shard -> shard.getTemporalRange().isPresent()).collect(toSet());
        return compactionSetCreator.createCompactionSets(tableInfo, temporalShards);
    }
    return compactionSetCreator.createCompactionSets(tableInfo, shardIndexInfos);
}

Example 3 with ShardMetadata

use of com.facebook.presto.raptor.metadata.ShardMetadata in project presto by prestodb.

the class TestShardOrganizerUtil method testGetOrganizationEligibleShards.

@Test
public void testGetOrganizationEligibleShards() throws Exception {
    int day1 = 1111;
    int day2 = 2222;
    SchemaTableName tableName = new SchemaTableName("default", "test");
    metadata.createTable(SESSION, tableMetadataBuilder(tableName).column("orderkey", BIGINT).column("orderdate", DATE).column("orderstatus", createVarcharType(3)).property("ordering", ImmutableList.of("orderstatus", "orderkey")).property("temporal_column", "orderdate").build());
    Table tableInfo = metadataDao.getTableInformation(tableName.getSchemaName(), tableName.getTableName());
    List<TableColumn> tableColumns = metadataDao.listTableColumns(tableInfo.getTableId());
    Map<String, TableColumn> tableColumnMap = Maps.uniqueIndex(tableColumns, TableColumn::getColumnName);
    long orderDate = tableColumnMap.get("orderdate").getColumnId();
    long orderKey = tableColumnMap.get("orderkey").getColumnId();
    long orderStatus = tableColumnMap.get("orderstatus").getColumnId();
    List<ShardInfo> shards = ImmutableList.<ShardInfo>builder().add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day1 + 10), new ColumnStats(orderKey, 13L, 14L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day2, day2 + 100), new ColumnStats(orderKey, 2L, 20L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day2), new ColumnStats(orderKey, 2L, 11L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day2), new ColumnStats(orderKey, 2L, null), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, null), new ColumnStats(orderKey, 2L, 11L), new ColumnStats(orderStatus, "aaa", "abc")))).build();
    long transactionId = shardManager.beginTransaction();
    shardManager.commitShards(transactionId, tableInfo.getTableId(), COLUMNS, shards, Optional.empty(), 0);
    Set<ShardMetadata> shardMetadatas = shardManager.getNodeShards("node1");
    Long temporalColumnId = metadataDao.getTemporalColumnId(tableInfo.getTableId());
    TableColumn temporalColumn = metadataDao.getTableColumn(tableInfo.getTableId(), temporalColumnId);
    Set<ShardIndexInfo> actual = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, false));
    List<ShardIndexInfo> expected = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.empty());
    assertEquals(actual, expected);
    List<TableColumn> sortColumns = metadataDao.listSortColumns(tableInfo.getTableId());
    Set<ShardIndexInfo> actualSortRange = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, true));
    List<ShardIndexInfo> expectedSortRange = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.of(sortColumns));
    assertEquals(actualSortRange, expectedSortRange);
}

Example 4 with ShardMetadata

use of com.facebook.presto.raptor.metadata.ShardMetadata in project presto by prestodb.

the class ShardRecoveryManager method enqueueMissingShards.

private synchronized void enqueueMissingShards() {
    try {
        for (ShardMetadata shard : getMissingShards()) {
            stats.incrementBackgroundShardRecovery();
            Futures.addCallback(shardQueue.submit(MissingShard.createBackgroundMissingShard(shard.getShardUuid(), shard.getCompressedSize())), failureCallback(t -> log.warn(t, "Error recovering shard: %s", shard.getShardUuid())));
        }
    } catch (Throwable t) {
        log.error(t, "Error creating shard recovery tasks");
    }
}

Example 5 with ShardMetadata

use of com.facebook.presto.raptor.metadata.ShardMetadata in project presto by prestodb.

the class ShardOrganizationManager method runOrganization.

private void runOrganization(long tableId) {
    Set<ShardMetadata> shardMetadatas = shardManager.getNodeShards(currentNodeIdentifier, tableId);
    Table tableInfo = metadataDao.getTableInformation(tableId);
    Set<ShardMetadata> filteredShards = shardMetadatas.stream().filter(shard -> !organizer.inProgress(shard.getShardUuid())).collect(toSet());
    Collection<ShardIndexInfo> indexInfos = getOrganizationEligibleShards(dbi, metadataDao, tableInfo, filteredShards, true);
    Set<OrganizationSet> organizationSets = createOrganizationSets(tableInfo, indexInfos);
    if (organizationSets.isEmpty()) {
        return;
    }
    log.info("Created %s organization set(s) from %s shards for table ID %s", organizationSets.size(), filteredShards.size(), tableId);
    long lastStartTime = System.currentTimeMillis();
    tablesInProgress.add(tableId);
    ImmutableList.Builder<CompletableFuture<?>> futures = ImmutableList.builder();
    for (OrganizationSet organizationSet : organizationSets) {
        futures.add(organizer.enqueue(organizationSet));
    }
    allAsList(futures.build()).whenComplete((value, throwable) -> {
        tablesInProgress.remove(tableId);
        organizerDao.updateLastStartTime(currentNodeIdentifier, tableId, lastStartTime);
    });
}