Examples with ShardMetadata io.trino.plugin.raptor.legacy.metadata.ShardMetadata used on opensource projects

Example 1 with ShardMetadata

use of io.trino.plugin.raptor.legacy.metadata.ShardMetadata in project trino by trinodb.

the class ShardRecoveryManager method enqueueMissingShards.

private synchronized void enqueueMissingShards() {
    try {
        for (ShardMetadata shard : getMissingShards()) {
            stats.incrementBackgroundShardRecovery();
            ListenableFuture<Void> future = shardQueue.submit(new MissingShard(shard.getShardUuid(), shard.getCompressedSize(), shard.getXxhash64(), false));
            addExceptionCallback(future, t -> log.warn(t, "Error recovering shard: %s", shard.getShardUuid()));
        }
    } catch (Throwable t) {
        log.error(t, "Error creating shard recovery tasks");
    }
}

Example 2 with ShardMetadata

use of io.trino.plugin.raptor.legacy.metadata.ShardMetadata in project trino by trinodb.

the class ShardRecoveryManager method recoverShard.

public Future<Void> recoverShard(UUID shardUuid) throws ExecutionException {
    ShardMetadata shard = shardManager.getShard(shardUuid);
    if (shard == null) {
        throw new TrinoException(RAPTOR_ERROR, "Shard does not exist in database: " + shardUuid);
    }
    stats.incrementActiveShardRecovery();
    return shardQueue.submit(new MissingShard(shardUuid, shard.getCompressedSize(), shard.getXxhash64(), true));
}

Example 3 with ShardMetadata

use of io.trino.plugin.raptor.legacy.metadata.ShardMetadata in project trino by trinodb.

the class TestShardOrganizerUtil method testGetOrganizationEligibleShards.

@Test
public void testGetOrganizationEligibleShards() {
    int day1 = 1111;
    int day2 = 2222;
    SchemaTableName tableName = new SchemaTableName("default", "test");
    metadata.createTable(SESSION, tableMetadataBuilder(tableName).column("orderkey", BIGINT).column("orderdate", DATE).column("orderstatus", createVarcharType(3)).property("ordering", ImmutableList.of("orderstatus", "orderkey")).property("temporal_column", "orderdate").build(), false);
    Table tableInfo = metadataDao.getTableInformation(tableName.getSchemaName(), tableName.getTableName());
    List<TableColumn> tableColumns = metadataDao.listTableColumns(tableInfo.getTableId());
    Map<String, TableColumn> tableColumnMap = Maps.uniqueIndex(tableColumns, TableColumn::getColumnName);
    long orderDate = tableColumnMap.get("orderdate").getColumnId();
    long orderKey = tableColumnMap.get("orderkey").getColumnId();
    long orderStatus = tableColumnMap.get("orderstatus").getColumnId();
    List<ShardInfo> shards = ImmutableList.<ShardInfo>builder().add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day1 + 10), new ColumnStats(orderKey, 13L, 14L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day2, day2 + 100), new ColumnStats(orderKey, 2L, 20L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day2), new ColumnStats(orderKey, 2L, 11L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day2), new ColumnStats(orderKey, 2L, null), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, null), new ColumnStats(orderKey, 2L, 11L), new ColumnStats(orderStatus, "aaa", "abc")))).build();
    long transactionId = shardManager.beginTransaction();
    shardManager.commitShards(transactionId, tableInfo.getTableId(), COLUMNS, shards, Optional.empty(), 0);
    Set<ShardMetadata> shardMetadatas = shardManager.getNodeShards("node1");
    Long temporalColumnId = metadataDao.getTemporalColumnId(tableInfo.getTableId());
    TableColumn temporalColumn = metadataDao.getTableColumn(tableInfo.getTableId(), temporalColumnId);
    Set<ShardIndexInfo> actual = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, false));
    List<ShardIndexInfo> expected = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.empty());
    assertEquals(actual, expected);
    List<TableColumn> sortColumns = metadataDao.listSortColumns(tableInfo.getTableId());
    Set<ShardIndexInfo> actualSortRange = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, true));
    List<ShardIndexInfo> expectedSortRange = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.of(sortColumns));
    assertEquals(actualSortRange, expectedSortRange);
}

Example 4 with ShardMetadata

use of io.trino.plugin.raptor.legacy.metadata.ShardMetadata in project trino by trinodb.

the class ShardEjector method process.

@VisibleForTesting
void process() {
    checkState(backupStore.isPresent(), "backup store must be present");
    // get the size of assigned shards for each node
    Map<String, Long> nodes = shardManager.getNodeBytes();
    Set<String> activeNodes = nodeSupplier.getWorkerNodes().stream().map(Node::getNodeIdentifier).collect(toSet());
    // only include active nodes
    nodes = new HashMap<>(filterKeys(nodes, activeNodes::contains));
    if (nodes.isEmpty()) {
        return;
    }
    // get current node size
    if (!nodes.containsKey(currentNode)) {
        return;
    }
    long nodeSize = nodes.get(currentNode);
    // get average node size
    long averageSize = round(nodes.values().stream().mapToLong(Long::longValue).average().getAsDouble());
    long maxSize = round(averageSize * 1.01);
    // skip if not above max
    if (nodeSize <= maxSize) {
        return;
    }
    // only include nodes that are below threshold
    nodes = new HashMap<>(filterValues(nodes, size -> size <= averageSize));
    // get non-bucketed node shards by size, largest to smallest
    List<ShardMetadata> shards = shardManager.getNodeShards(currentNode).stream().filter(shard -> shard.getBucketNumber().isEmpty()).sorted(comparingLong(ShardMetadata::getCompressedSize).reversed()).collect(toList());
    // eject shards while current node is above max
    Queue<ShardMetadata> queue = new ArrayDeque<>(shards);
    while ((nodeSize > maxSize) && !queue.isEmpty()) {
        ShardMetadata shard = queue.remove();
        long shardSize = shard.getCompressedSize();
        UUID shardUuid = shard.getShardUuid();
        // verify backup exists
        if (!backupStore.get().shardExists(shardUuid)) {
            log.warn("No backup for shard: %s", shardUuid);
        }
        // pick target node
        String target = pickTargetNode(nodes, shardSize, averageSize);
        if (target == null) {
            return;
        }
        long targetSize = nodes.get(target);
        // stats
        log.info("Moving shard %s to node %s (shard: %s, node: %s, average: %s, target: %s)", shardUuid, target, shardSize, nodeSize, averageSize, targetSize);
        shardsEjected.update(1);
        // update size
        nodes.put(target, targetSize + shardSize);
        nodeSize -= shardSize;
        // move assignment
        shardManager.replaceShardAssignment(shard.getTableId(), shardUuid, target, false);
        // delete local file
        File file = storageService.getStorageFile(shardUuid);
        if (file.exists() && !file.delete()) {
            log.warn("Failed to delete shard file: %s", file);
        }
    }
}

Example 5 with ShardMetadata

use of io.trino.plugin.raptor.legacy.metadata.ShardMetadata in project trino by trinodb.

the class ShardOrganizerUtil method getOrganizationEligibleShards.

public static Collection<ShardIndexInfo> getOrganizationEligibleShards(Jdbi dbi, MetadataDao metadataDao, Table tableInfo, Collection<ShardMetadata> shards, boolean includeSortColumns) {
    Map<Long, ShardMetadata> shardsById = uniqueIndex(shards, ShardMetadata::getShardId);
    long tableId = tableInfo.getTableId();
    ImmutableList.Builder<String> columnsBuilder = ImmutableList.builder();
    columnsBuilder.add("shard_id");
    // include temporal columns if present
    Optional<TableColumn> temporalColumn = Optional.empty();
    if (tableInfo.getTemporalColumnId().isPresent()) {
        long temporalColumnId = tableInfo.getTemporalColumnId().getAsLong();
        temporalColumn = Optional.of(metadataDao.getTableColumn(tableId, temporalColumnId));
        columnsBuilder.add(minColumn(temporalColumnId), maxColumn(temporalColumnId));
    }
    // include sort columns if needed
    Optional<List<TableColumn>> sortColumns = Optional.empty();
    if (includeSortColumns) {
        sortColumns = Optional.of(metadataDao.listSortColumns(tableId));
        for (TableColumn column : sortColumns.get()) {
            columnsBuilder.add(minColumn(column.getColumnId()), maxColumn(column.getColumnId()));
        }
    }
    String columnToSelect = Joiner.on(",\n").join(columnsBuilder.build());
    ImmutableList.Builder<ShardIndexInfo> indexInfoBuilder = ImmutableList.builder();
    try (Connection connection = dbi.open().getConnection()) {
        for (List<ShardMetadata> partitionedShards : partition(shards, 1000)) {
            String shardIds = Joiner.on(",").join(nCopies(partitionedShards.size(), "?"));
            String sql = format("" + "SELECT %s\n" + "FROM %s\n" + "WHERE shard_id IN (%s)", columnToSelect, shardIndexTable(tableId), shardIds);
            try (PreparedStatement statement = connection.prepareStatement(sql)) {
                for (int i = 0; i < partitionedShards.size(); i++) {
                    statement.setLong(i + 1, partitionedShards.get(i).getShardId());
                }
                try (ResultSet resultSet = statement.executeQuery()) {
                    while (resultSet.next()) {
                        long shardId = resultSet.getLong("shard_id");
                        Optional<ShardRange> sortRange = Optional.empty();
                        if (includeSortColumns) {
                            sortRange = getShardRange(sortColumns.get(), resultSet);
                            if (sortRange.isEmpty()) {
                                continue;
                            }
                        }
                        Optional<ShardRange> temporalRange = Optional.empty();
                        if (temporalColumn.isPresent()) {
                            temporalRange = getShardRange(ImmutableList.of(temporalColumn.get()), resultSet);
                            if (temporalRange.isEmpty()) {
                                continue;
                            }
                        }
                        ShardMetadata shardMetadata = shardsById.get(shardId);
                        indexInfoBuilder.add(toShardIndexInfo(shardMetadata, temporalRange, sortRange));
                    }
                }
            }
        }
    } catch (SQLException e) {
        throw new RuntimeException(e);
    }
    return indexInfoBuilder.build();
}