Examples with Table io.trino.plugin.raptor.legacy.metadata.Table used on opensource projects

Example 1 with Table

use of io.trino.plugin.raptor.legacy.metadata.Table in project trino by trinodb.

the class TestShardOrganizerUtil method testGetOrganizationEligibleShards.

@Test
public void testGetOrganizationEligibleShards() {
    int day1 = 1111;
    int day2 = 2222;
    SchemaTableName tableName = new SchemaTableName("default", "test");
    metadata.createTable(SESSION, tableMetadataBuilder(tableName).column("orderkey", BIGINT).column("orderdate", DATE).column("orderstatus", createVarcharType(3)).property("ordering", ImmutableList.of("orderstatus", "orderkey")).property("temporal_column", "orderdate").build(), false);
    Table tableInfo = metadataDao.getTableInformation(tableName.getSchemaName(), tableName.getTableName());
    List<TableColumn> tableColumns = metadataDao.listTableColumns(tableInfo.getTableId());
    Map<String, TableColumn> tableColumnMap = Maps.uniqueIndex(tableColumns, TableColumn::getColumnName);
    long orderDate = tableColumnMap.get("orderdate").getColumnId();
    long orderKey = tableColumnMap.get("orderkey").getColumnId();
    long orderStatus = tableColumnMap.get("orderstatus").getColumnId();
    List<ShardInfo> shards = ImmutableList.<ShardInfo>builder().add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day1 + 10), new ColumnStats(orderKey, 13L, 14L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day2, day2 + 100), new ColumnStats(orderKey, 2L, 20L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day2), new ColumnStats(orderKey, 2L, 11L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day2), new ColumnStats(orderKey, 2L, null), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, null), new ColumnStats(orderKey, 2L, 11L), new ColumnStats(orderStatus, "aaa", "abc")))).build();
    long transactionId = shardManager.beginTransaction();
    shardManager.commitShards(transactionId, tableInfo.getTableId(), COLUMNS, shards, Optional.empty(), 0);
    Set<ShardMetadata> shardMetadatas = shardManager.getNodeShards("node1");
    Long temporalColumnId = metadataDao.getTemporalColumnId(tableInfo.getTableId());
    TableColumn temporalColumn = metadataDao.getTableColumn(tableInfo.getTableId(), temporalColumnId);
    Set<ShardIndexInfo> actual = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, false));
    List<ShardIndexInfo> expected = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.empty());
    assertEquals(actual, expected);
    List<TableColumn> sortColumns = metadataDao.listSortColumns(tableInfo.getTableId());
    Set<ShardIndexInfo> actualSortRange = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, true));
    List<ShardIndexInfo> expectedSortRange = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.of(sortColumns));
    assertEquals(actualSortRange, expectedSortRange);
}

Example 2 with Table

use of io.trino.plugin.raptor.legacy.metadata.Table in project trino by trinodb.

the class ShardCompactionManager method filterAndCreateCompactionSets.

private Collection<OrganizationSet> filterAndCreateCompactionSets(long tableId, Collection<ShardMetadata> tableShards) {
    Table tableInfo = metadataDao.getTableInformation(tableId);
    OptionalLong temporalColumnId = tableInfo.getTemporalColumnId();
    if (temporalColumnId.isPresent()) {
        TableColumn tableColumn = metadataDao.getTableColumn(tableId, temporalColumnId.getAsLong());
        if (!isValidTemporalColumn(tableId, tableColumn.getDataType())) {
            return ImmutableSet.of();
        }
    }
    Set<ShardMetadata> filteredShards = tableShards.stream().filter(this::needsCompaction).filter(shard -> !organizer.inProgress(shard.getShardUuid())).collect(toSet());
    Collection<ShardIndexInfo> shardIndexInfos = getOrganizationEligibleShards(dbi, metadataDao, tableInfo, filteredShards, false);
    if (tableInfo.getTemporalColumnId().isPresent()) {
        Set<ShardIndexInfo> temporalShards = shardIndexInfos.stream().filter(shard -> shard.getTemporalRange().isPresent()).collect(toSet());
        return compactionSetCreator.createCompactionSets(tableInfo, temporalShards);
    }
    return compactionSetCreator.createCompactionSets(tableInfo, shardIndexInfos);
}

Example 3 with Table

use of io.trino.plugin.raptor.legacy.metadata.Table in project trino by trinodb.

the class ShardOrganizationManager method runOrganization.

private void runOrganization(long tableId) {
    Set<ShardMetadata> shardMetadatas = shardManager.getNodeShards(currentNodeIdentifier, tableId);
    Table tableInfo = metadataDao.getTableInformation(tableId);
    Set<ShardMetadata> filteredShards = shardMetadatas.stream().filter(shard -> !organizer.inProgress(shard.getShardUuid())).collect(toSet());
    Collection<ShardIndexInfo> indexInfos = getOrganizationEligibleShards(dbi, metadataDao, tableInfo, filteredShards, true);
    Set<OrganizationSet> organizationSets = createOrganizationSets(tableInfo, indexInfos);
    if (organizationSets.isEmpty()) {
        return;
    }
    log.info("Created %s organization set(s) from %s shards for table ID %s", organizationSets.size(), filteredShards.size(), tableId);
    long lastStartTime = System.currentTimeMillis();
    tablesInProgress.add(tableId);
    ImmutableList.Builder<CompletableFuture<?>> futures = ImmutableList.builder();
    for (OrganizationSet organizationSet : organizationSets) {
        futures.add(organizer.enqueue(organizationSet));
    }
    allAsList(futures.build()).whenComplete((value, throwable) -> {
        tablesInProgress.remove(tableId);
        organizerDao.updateLastStartTime(currentNodeIdentifier, tableId, lastStartTime);
    });
}

Example 4 with Table

use of io.trino.plugin.raptor.legacy.metadata.Table in project trino by trinodb.

the class RaptorMetadata method getTableHandle.

private RaptorTableHandle getTableHandle(SchemaTableName tableName) {
    requireNonNull(tableName, "tableName is null");
    Table table = dao.getTableInformation(tableName.getSchemaName(), tableName.getTableName());
    if (table == null) {
        return null;
    }
    List<TableColumn> tableColumns = dao.listTableColumns(table.getTableId());
    checkArgument(!tableColumns.isEmpty(), "Table '%s' does not have any columns", tableName);
    return new RaptorTableHandle(tableName.getSchemaName(), tableName.getTableName(), table.getTableId(), table.getDistributionId(), table.getDistributionName(), table.getBucketCount(), table.isOrganized(), OptionalLong.empty(), TupleDomain.all(), table.getDistributionId().map(shardManager::getBucketAssignments), false);
}

Example 5 with Table

use of io.trino.plugin.raptor.legacy.metadata.Table in project trino by trinodb.

the class RaptorMetadata method finishDelete.

@Override
public void finishDelete(ConnectorSession session, ConnectorTableHandle tableHandle, Collection<Slice> fragments) {
    RaptorTableHandle table = (RaptorTableHandle) tableHandle;
    long transactionId = table.getTransactionId().getAsLong();
    long tableId = table.getTableId();
    List<ColumnInfo> columns = getColumnHandles(session, tableHandle).values().stream().map(RaptorColumnHandle.class::cast).map(ColumnInfo::fromHandle).collect(toList());
    ImmutableSet.Builder<UUID> oldShardUuidsBuilder = ImmutableSet.builder();
    ImmutableList.Builder<ShardInfo> newShardsBuilder = ImmutableList.builder();
    fragments.stream().map(fragment -> SHARD_DELTA_CODEC.fromJson(fragment.getBytes())).forEach(delta -> {
        oldShardUuidsBuilder.addAll(delta.getOldShardUuids());
        newShardsBuilder.addAll(delta.getNewShards());
    });
    Set<UUID> oldShardUuids = oldShardUuidsBuilder.build();
    List<ShardInfo> newShards = newShardsBuilder.build();
    OptionalLong updateTime = OptionalLong.of(session.getStart().toEpochMilli());
    log.info("Finishing delete for tableId %s (removed: %s, rewritten: %s)", tableId, oldShardUuids.size() - newShards.size(), newShards.size());
    shardManager.replaceShardUuids(transactionId, tableId, columns, oldShardUuids, newShards, updateTime);
    clearRollback();
}