Examples with Table com.facebook.presto.raptor.metadata.Table used on opensource projects

Example 1 with Table

use of com.facebook.presto.raptor.metadata.Table in project presto by prestodb.

the class TestShardOrganizerUtil method testGetOrganizationEligibleShards.

@Test
public void testGetOrganizationEligibleShards() {
    int day1 = 1111;
    int day2 = 2222;
    SchemaTableName tableName = new SchemaTableName("default", "test");
    metadata.createTable(SESSION, tableMetadataBuilder(tableName).column("orderkey", BIGINT).column("orderdate", DATE).column("orderstatus", createVarcharType(3)).property("ordering", ImmutableList.of("orderstatus", "orderkey")).property("temporal_column", "orderdate").property("table_supports_delta_delete", false).build(), false);
    Table tableInfo = metadataDao.getTableInformation(tableName.getSchemaName(), tableName.getTableName());
    List<TableColumn> tableColumns = metadataDao.listTableColumns(tableInfo.getTableId());
    Map<String, TableColumn> tableColumnMap = Maps.uniqueIndex(tableColumns, TableColumn::getColumnName);
    long orderDate = tableColumnMap.get("orderdate").getColumnId();
    long orderKey = tableColumnMap.get("orderkey").getColumnId();
    long orderStatus = tableColumnMap.get("orderstatus").getColumnId();
    List<ShardInfo> shards = ImmutableList.<ShardInfo>builder().add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day1 + 10), new ColumnStats(orderKey, 13L, 14L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day2, day2 + 100), new ColumnStats(orderKey, 2L, 20L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day2), new ColumnStats(orderKey, 2L, 11L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day2), new ColumnStats(orderKey, 2L, null), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, null), new ColumnStats(orderKey, 2L, 11L), new ColumnStats(orderStatus, "aaa", "abc")))).build();
    long transactionId = shardManager.beginTransaction();
    shardManager.commitShards(transactionId, tableInfo.getTableId(), COLUMNS, shards, Optional.empty(), 0);
    Set<ShardMetadata> shardMetadatas = shardManager.getNodeShardsAndDeltas("node1");
    Long temporalColumnId = metadataDao.getTemporalColumnId(tableInfo.getTableId());
    TableColumn temporalColumn = metadataDao.getTableColumn(tableInfo.getTableId(), temporalColumnId);
    Set<ShardIndexInfo> actual = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, false));
    List<ShardIndexInfo> expected = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.empty());
    assertEquals(actual, expected);
    List<TableColumn> sortColumns = metadataDao.listSortColumns(tableInfo.getTableId());
    Set<ShardIndexInfo> actualSortRange = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, true));
    List<ShardIndexInfo> expectedSortRange = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.of(sortColumns));
    assertEquals(actualSortRange, expectedSortRange);
}

Example 2 with Table

use of com.facebook.presto.raptor.metadata.Table in project presto by prestodb.

the class RaptorMetadata method getTableHandle.

private RaptorTableHandle getTableHandle(SchemaTableName tableName) {
    requireNonNull(tableName, "tableName is null");
    Table table = dao.getTableInformation(tableName.getSchemaName(), tableName.getTableName());
    if (table == null) {
        return null;
    }
    List<TableColumn> tableColumns = dao.listTableColumns(table.getTableId());
    checkArgument(!tableColumns.isEmpty(), "Table %s does not have any columns", tableName);
    return new RaptorTableHandle(connectorId, tableName.getSchemaName(), tableName.getTableName(), table.getTableId(), table.getDistributionId(), table.getDistributionName(), table.getBucketCount(), table.isOrganized(), OptionalLong.empty(), Optional.empty(), false, table.isTableSupportsDeltaDelete());
}

Example 3 with Table

use of com.facebook.presto.raptor.metadata.Table in project presto by prestodb.

the class ShardCompactionManager method filterAndCreateCompactionSets.

private Collection<OrganizationSet> filterAndCreateCompactionSets(long tableId, Collection<ShardMetadata> tableShards) {
    Table tableInfo = metadataDao.getTableInformation(tableId);
    OptionalLong temporalColumnId = tableInfo.getTemporalColumnId();
    if (temporalColumnId.isPresent()) {
        TableColumn tableColumn = metadataDao.getTableColumn(tableId, temporalColumnId.getAsLong());
        if (!isValidTemporalColumn(tableId, tableColumn.getDataType())) {
            return ImmutableSet.of();
        }
    }
    Set<ShardMetadata> filteredShards = tableShards.stream().filter(this::needsCompaction).filter(shard -> !organizer.inProgress(shard.getShardUuid())).collect(toSet());
    Collection<ShardIndexInfo> shardIndexInfos = getOrganizationEligibleShards(dbi, metadataDao, tableInfo, filteredShards, false);
    if (tableInfo.getTemporalColumnId().isPresent()) {
        Set<ShardIndexInfo> temporalShards = shardIndexInfos.stream().filter(shard -> shard.getTemporalRange().isPresent()).collect(toSet());
        return compactionSetCreator.createCompactionSets(tableInfo, temporalShards);
    }
    return compactionSetCreator.createCompactionSets(tableInfo, shardIndexInfos);
}

Example 4 with Table

use of com.facebook.presto.raptor.metadata.Table in project presto by prestodb.

the class ShardOrganizationManager method runOrganization.

private void runOrganization(long tableId) {
    Set<ShardMetadata> shardMetadatas = shardManager.getNodeShards(currentNodeIdentifier, tableId);
    Table tableInfo = metadataDao.getTableInformation(tableId);
    Set<ShardMetadata> filteredShards = shardMetadatas.stream().filter(shard -> !organizer.inProgress(shard.getShardUuid())).collect(toSet());
    Collection<ShardIndexInfo> indexInfos = getOrganizationEligibleShards(dbi, metadataDao, tableInfo, filteredShards, true);
    Set<OrganizationSet> organizationSets = createOrganizationSets(temporalFunction, tableInfo, indexInfos);
    if (organizationSets.isEmpty()) {
        return;
    }
    log.info("Created %s organization set(s) from %s shards for table ID %s", organizationSets.size(), filteredShards.size(), tableId);
    long lastStartTime = System.currentTimeMillis();
    tablesInProgress.add(tableId);
    ImmutableList.Builder<ListenableFuture<?>> futures = ImmutableList.builder();
    for (OrganizationSet organizationSet : organizationSets) {
        futures.add(organizer.enqueue(organizationSet));
    }
    futures.build().forEach(listenableFuture -> listenableFuture.addListener(() -> {
        tablesInProgress.remove(tableId);
        organizerDao.updateLastStartTime(currentNodeIdentifier, tableId, lastStartTime);
    }, MoreExecutors.directExecutor()));
}

Example 5 with Table

use of com.facebook.presto.raptor.metadata.Table in project presto by prestodb.

the class RaptorMetadata method finishDelete.

@Override
public void finishDelete(ConnectorSession session, ConnectorTableHandle tableHandle, Collection<Slice> fragments) {
    RaptorTableHandle table = (RaptorTableHandle) tableHandle;
    long transactionId = table.getTransactionId().getAsLong();
    long tableId = table.getTableId();
    List<ColumnInfo> columns = getColumnHandles(session, tableHandle).values().stream().map(RaptorColumnHandle.class::cast).map(ColumnInfo::fromHandle).collect(toList());
    if (table.isTableSupportsDeltaDelete()) {
        ImmutableMap.Builder<UUID, DeltaInfoPair> shardMapBuilder = ImmutableMap.builder();
        fragments.stream().map(fragment -> SHARD_DELETE_DELTA_CODEC.fromJson(fragment.getBytes())).forEach(delta -> shardMapBuilder.put(delta.getOldShardUuid(), delta.getDeltaInfoPair()));
        OptionalLong updateTime = OptionalLong.of(session.getStartTime());
        log.info("Finishing delete for tableId %s (affected shardUuid: %s)", tableId, shardMapBuilder.build().size());
        shardManager.replaceDeltaUuids(transactionId, tableId, columns, shardMapBuilder.build(), updateTime);
    } else {
        ImmutableSet.Builder<UUID> oldShardUuidsBuilder = ImmutableSet.builder();
        ImmutableList.Builder<ShardInfo> newShardsBuilder = ImmutableList.builder();
        fragments.stream().map(fragment -> SHARD_DELTA_CODEC.fromJson(fragment.getBytes())).forEach(delta -> {
            oldShardUuidsBuilder.addAll(delta.getOldShardUuids());
            newShardsBuilder.addAll(delta.getNewShards());
        });
        Set<UUID> oldShardUuids = oldShardUuidsBuilder.build();
        List<ShardInfo> newShards = newShardsBuilder.build();
        OptionalLong updateTime = OptionalLong.of(session.getStartTime());
        log.info("Finishing delete for tableId %s (removed: %s, rewritten: %s)", tableId, oldShardUuids.size() - newShards.size(), newShards.size());
        shardManager.replaceShardUuids(transactionId, tableId, columns, oldShardUuids, newShards, updateTime);
    }
    clearRollback();
}