Examples with TableColumn io.trino.plugin.raptor.legacy.metadata.TableColumn used on opensource projects

Example 1 with TableColumn

use of io.trino.plugin.raptor.legacy.metadata.TableColumn in project trino by trinodb.

the class RaptorMetadata method getColumnHandles.

@Override
public Map<String, ColumnHandle> getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) {
    RaptorTableHandle raptorTableHandle = (RaptorTableHandle) tableHandle;
    ImmutableMap.Builder<String, ColumnHandle> builder = ImmutableMap.builder();
    for (TableColumn tableColumn : dao.listTableColumns(raptorTableHandle.getTableId())) {
        builder.put(tableColumn.getColumnName(), getRaptorColumnHandle(tableColumn));
    }
    RaptorColumnHandle uuidColumn = shardUuidColumnHandle();
    builder.put(uuidColumn.getColumnName(), uuidColumn);
    if (raptorTableHandle.isBucketed()) {
        RaptorColumnHandle bucketNumberColumn = bucketNumberColumnHandle();
        builder.put(bucketNumberColumn.getColumnName(), bucketNumberColumn);
    }
    return builder.buildOrThrow();
}

Example 2 with TableColumn

use of io.trino.plugin.raptor.legacy.metadata.TableColumn in project trino by trinodb.

the class RaptorMetadata method getTableMetadata.

@Override
public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle tableHandle) {
    RaptorTableHandle handle = (RaptorTableHandle) tableHandle;
    SchemaTableName tableName = new SchemaTableName(handle.getSchemaName(), handle.getTableName());
    List<TableColumn> tableColumns = dao.listTableColumns(handle.getTableId());
    if (tableColumns.isEmpty()) {
        throw new TableNotFoundException(tableName);
    }
    ImmutableMap.Builder<String, Object> properties = ImmutableMap.builder();
    SortedMap<Integer, String> bucketing = new TreeMap<>();
    SortedMap<Integer, String> ordering = new TreeMap<>();
    for (TableColumn column : tableColumns) {
        if (column.isTemporal()) {
            properties.put(TEMPORAL_COLUMN_PROPERTY, column.getColumnName());
        }
        column.getBucketOrdinal().ifPresent(bucketOrdinal -> bucketing.put(bucketOrdinal, column.getColumnName()));
        column.getSortOrdinal().ifPresent(sortOrdinal -> ordering.put(sortOrdinal, column.getColumnName()));
    }
    if (!bucketing.isEmpty()) {
        properties.put(BUCKETED_ON_PROPERTY, ImmutableList.copyOf(bucketing.values()));
    }
    if (!ordering.isEmpty()) {
        properties.put(ORDERING_PROPERTY, ImmutableList.copyOf(ordering.values()));
    }
    handle.getBucketCount().ifPresent(bucketCount -> properties.put(BUCKET_COUNT_PROPERTY, bucketCount));
    handle.getDistributionName().ifPresent(distributionName -> properties.put(DISTRIBUTION_NAME_PROPERTY, distributionName));
    // Only display organization property if set
    if (handle.isOrganized()) {
        properties.put(ORGANIZED_PROPERTY, true);
    }
    List<ColumnMetadata> columns = tableColumns.stream().map(TableColumn::toColumnMetadata).collect(toCollection(ArrayList::new));
    columns.add(hiddenColumn(SHARD_UUID_COLUMN_NAME, SHARD_UUID_COLUMN_TYPE));
    if (handle.isBucketed()) {
        columns.add(hiddenColumn(BUCKET_NUMBER_COLUMN_NAME, INTEGER));
    }
    return new ConnectorTableMetadata(tableName, columns, properties.buildOrThrow());
}

Example 3 with TableColumn

use of io.trino.plugin.raptor.legacy.metadata.TableColumn in project trino by trinodb.

the class ShardOrganizerUtil method getShardRange.

private static Optional<ShardRange> getShardRange(List<TableColumn> columns, ResultSet resultSet) throws SQLException {
    ImmutableList.Builder<Object> minValuesBuilder = ImmutableList.builder();
    ImmutableList.Builder<Object> maxValuesBuilder = ImmutableList.builder();
    ImmutableList.Builder<Type> typeBuilder = ImmutableList.builder();
    for (TableColumn tableColumn : columns) {
        long columnId = tableColumn.getColumnId();
        Type type = tableColumn.getDataType();
        Object min = getValue(resultSet, type, minColumn(columnId));
        Object max = getValue(resultSet, type, maxColumn(columnId));
        if (min == null || max == null) {
            return Optional.empty();
        }
        minValuesBuilder.add(min);
        maxValuesBuilder.add(max);
        typeBuilder.add(type);
    }
    List<Type> types = typeBuilder.build();
    Tuple minTuple = new Tuple(types, minValuesBuilder.build());
    Tuple maxTuple = new Tuple(types, maxValuesBuilder.build());
    return Optional.of(ShardRange.of(minTuple, maxTuple));
}

Example 4 with TableColumn

use of io.trino.plugin.raptor.legacy.metadata.TableColumn in project trino by trinodb.

the class TestShardOrganizerUtil method testGetOrganizationEligibleShards.

@Test
public void testGetOrganizationEligibleShards() {
    int day1 = 1111;
    int day2 = 2222;
    SchemaTableName tableName = new SchemaTableName("default", "test");
    metadata.createTable(SESSION, tableMetadataBuilder(tableName).column("orderkey", BIGINT).column("orderdate", DATE).column("orderstatus", createVarcharType(3)).property("ordering", ImmutableList.of("orderstatus", "orderkey")).property("temporal_column", "orderdate").build(), false);
    Table tableInfo = metadataDao.getTableInformation(tableName.getSchemaName(), tableName.getTableName());
    List<TableColumn> tableColumns = metadataDao.listTableColumns(tableInfo.getTableId());
    Map<String, TableColumn> tableColumnMap = Maps.uniqueIndex(tableColumns, TableColumn::getColumnName);
    long orderDate = tableColumnMap.get("orderdate").getColumnId();
    long orderKey = tableColumnMap.get("orderkey").getColumnId();
    long orderStatus = tableColumnMap.get("orderstatus").getColumnId();
    List<ShardInfo> shards = ImmutableList.<ShardInfo>builder().add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day1 + 10), new ColumnStats(orderKey, 13L, 14L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day2, day2 + 100), new ColumnStats(orderKey, 2L, 20L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day2), new ColumnStats(orderKey, 2L, 11L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day2), new ColumnStats(orderKey, 2L, null), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, null), new ColumnStats(orderKey, 2L, 11L), new ColumnStats(orderStatus, "aaa", "abc")))).build();
    long transactionId = shardManager.beginTransaction();
    shardManager.commitShards(transactionId, tableInfo.getTableId(), COLUMNS, shards, Optional.empty(), 0);
    Set<ShardMetadata> shardMetadatas = shardManager.getNodeShards("node1");
    Long temporalColumnId = metadataDao.getTemporalColumnId(tableInfo.getTableId());
    TableColumn temporalColumn = metadataDao.getTableColumn(tableInfo.getTableId(), temporalColumnId);
    Set<ShardIndexInfo> actual = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, false));
    List<ShardIndexInfo> expected = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.empty());
    assertEquals(actual, expected);
    List<TableColumn> sortColumns = metadataDao.listSortColumns(tableInfo.getTableId());
    Set<ShardIndexInfo> actualSortRange = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, true));
    List<ShardIndexInfo> expectedSortRange = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.of(sortColumns));
    assertEquals(actualSortRange, expectedSortRange);
}

Example 5 with TableColumn

use of io.trino.plugin.raptor.legacy.metadata.TableColumn in project trino by trinodb.

the class TestShardOrganizerUtil method getShardIndexInfo.

private static List<ShardIndexInfo> getShardIndexInfo(Table tableInfo, List<ShardInfo> shards, TableColumn temporalColumn, Optional<List<TableColumn>> sortColumns) {
    long tableId = tableInfo.getTableId();
    Type temporalType = temporalColumn.getDataType();
    ImmutableList.Builder<ShardIndexInfo> builder = ImmutableList.builder();
    for (ShardInfo shard : shards) {
        ColumnStats temporalColumnStats = shard.getColumnStats().stream().filter(columnStats -> columnStats.getColumnId() == temporalColumn.getColumnId()).findFirst().get();
        if (temporalColumnStats.getMin() == null || temporalColumnStats.getMax() == null) {
            continue;
        }
        Optional<ShardRange> sortRange = Optional.empty();
        if (sortColumns.isPresent()) {
            Map<Long, ColumnStats> columnIdToStats = Maps.uniqueIndex(shard.getColumnStats(), ColumnStats::getColumnId);
            ImmutableList.Builder<Type> typesBuilder = ImmutableList.builder();
            ImmutableList.Builder<Object> minBuilder = ImmutableList.builder();
            ImmutableList.Builder<Object> maxBuilder = ImmutableList.builder();
            boolean isShardEligible = true;
            for (TableColumn sortColumn : sortColumns.get()) {
                ColumnStats columnStats = columnIdToStats.get(sortColumn.getColumnId());
                typesBuilder.add(sortColumn.getDataType());
                if (columnStats.getMin() == null || columnStats.getMax() == null) {
                    isShardEligible = false;
                    break;
                }
                minBuilder.add(columnStats.getMin());
                maxBuilder.add(columnStats.getMax());
            }
            if (!isShardEligible) {
                continue;
            }
            List<Type> types = typesBuilder.build();
            List<Object> minValues = minBuilder.build();
            List<Object> maxValues = maxBuilder.build();
            sortRange = Optional.of(ShardRange.of(new Tuple(types, minValues), new Tuple(types, maxValues)));
        }
        builder.add(new ShardIndexInfo(tableId, OptionalInt.empty(), shard.getShardUuid(), shard.getRowCount(), shard.getUncompressedSize(), sortRange, Optional.of(ShardRange.of(new Tuple(temporalType, temporalColumnStats.getMin()), new Tuple(temporalType, temporalColumnStats.getMax())))));
    }
    return builder.build();
}