Examples with ColumnStats io.trino.plugin.raptor.legacy.metadata.ColumnStats used on opensource projects

Example 1 with ColumnStats

use of io.trino.plugin.raptor.legacy.metadata.ColumnStats in project trino by trinodb.

the class ShardStats method indexBoolean.

private static ColumnStats indexBoolean(OrcRecordReader reader, long columnId) throws IOException {
    boolean minSet = false;
    boolean maxSet = false;
    boolean min = false;
    boolean max = false;
    while (true) {
        Page page = reader.nextPage();
        if (page == null) {
            break;
        }
        Block block = page.getBlock(0).getLoadedBlock();
        for (int i = 0; i < page.getPositionCount(); i++) {
            if (block.isNull(i)) {
                continue;
            }
            boolean value = BooleanType.BOOLEAN.getBoolean(block, i);
            if (!minSet || Boolean.compare(value, min) < 0) {
                minSet = true;
                min = value;
            }
            if (!maxSet || Boolean.compare(value, max) > 0) {
                maxSet = true;
                max = value;
            }
        }
    }
    return new ColumnStats(columnId, minSet ? min : null, maxSet ? max : null);
}

Example 2 with ColumnStats

use of io.trino.plugin.raptor.legacy.metadata.ColumnStats in project trino by trinodb.

the class ShardStats method indexString.

private static ColumnStats indexString(Type type, OrcRecordReader reader, long columnId) throws IOException {
    boolean minSet = false;
    boolean maxSet = false;
    Slice min = null;
    Slice max = null;
    while (true) {
        Page page = reader.nextPage();
        if (page == null) {
            break;
        }
        Block block = page.getBlock(0).getLoadedBlock();
        for (int i = 0; i < page.getPositionCount(); i++) {
            if (block.isNull(i)) {
                continue;
            }
            Slice slice = type.getSlice(block, i);
            slice = truncateIndexValue(slice);
            if (!minSet || (slice.compareTo(min) < 0)) {
                minSet = true;
                min = slice;
            }
            if (!maxSet || (slice.compareTo(max) > 0)) {
                maxSet = true;
                max = slice;
            }
        }
    }
    return new ColumnStats(columnId, minSet ? min.toStringUtf8() : null, maxSet ? max.toStringUtf8() : null);
}

Example 3 with ColumnStats

use of io.trino.plugin.raptor.legacy.metadata.ColumnStats in project trino by trinodb.

the class ShardStats method indexDouble.

private static ColumnStats indexDouble(OrcRecordReader reader, long columnId) throws IOException {
    boolean minSet = false;
    boolean maxSet = false;
    double min = 0;
    double max = 0;
    while (true) {
        Page page = reader.nextPage();
        if (page == null) {
            break;
        }
        Block block = page.getBlock(0).getLoadedBlock();
        for (int i = 0; i < page.getPositionCount(); i++) {
            if (block.isNull(i)) {
                continue;
            }
            double value = DoubleType.DOUBLE.getDouble(block, i);
            if (isNaN(value)) {
                continue;
            }
            if (value == -0.0) {
                value = 0.0;
            }
            if (!minSet || (value < min)) {
                minSet = true;
                min = value;
            }
            if (!maxSet || (value > max)) {
                maxSet = true;
                max = value;
            }
        }
    }
    if (isInfinite(min)) {
        minSet = false;
    }
    if (isInfinite(max)) {
        maxSet = false;
    }
    return new ColumnStats(columnId, minSet ? min : null, maxSet ? max : null);
}

Example 4 with ColumnStats

use of io.trino.plugin.raptor.legacy.metadata.ColumnStats in project trino by trinodb.

the class RaptorStorageManager method computeShardStats.

private List<ColumnStats> computeShardStats(File file) {
    try (OrcDataSource dataSource = fileOrcDataSource(orcReaderOptions, file)) {
        OrcReader reader = OrcReader.createOrcReader(dataSource, orcReaderOptions).orElseThrow(() -> new TrinoException(RAPTOR_ERROR, "Data file is empty: " + file));
        ImmutableList.Builder<ColumnStats> list = ImmutableList.builder();
        for (ColumnInfo info : getColumnInfo(reader)) {
            computeColumnStats(reader, info.getColumnId(), info.getType(), typeManager).ifPresent(list::add);
        }
        return list.build();
    } catch (IOException e) {
        throw new TrinoException(RAPTOR_ERROR, "Failed to read file: " + file, e);
    }
}

Example 5 with ColumnStats

use of io.trino.plugin.raptor.legacy.metadata.ColumnStats in project trino by trinodb.

the class TestShardOrganizerUtil method testGetOrganizationEligibleShards.

@Test
public void testGetOrganizationEligibleShards() {
    int day1 = 1111;
    int day2 = 2222;
    SchemaTableName tableName = new SchemaTableName("default", "test");
    metadata.createTable(SESSION, tableMetadataBuilder(tableName).column("orderkey", BIGINT).column("orderdate", DATE).column("orderstatus", createVarcharType(3)).property("ordering", ImmutableList.of("orderstatus", "orderkey")).property("temporal_column", "orderdate").build(), false);
    Table tableInfo = metadataDao.getTableInformation(tableName.getSchemaName(), tableName.getTableName());
    List<TableColumn> tableColumns = metadataDao.listTableColumns(tableInfo.getTableId());
    Map<String, TableColumn> tableColumnMap = Maps.uniqueIndex(tableColumns, TableColumn::getColumnName);
    long orderDate = tableColumnMap.get("orderdate").getColumnId();
    long orderKey = tableColumnMap.get("orderkey").getColumnId();
    long orderStatus = tableColumnMap.get("orderstatus").getColumnId();
    List<ShardInfo> shards = ImmutableList.<ShardInfo>builder().add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day1 + 10), new ColumnStats(orderKey, 13L, 14L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day2, day2 + 100), new ColumnStats(orderKey, 2L, 20L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day2), new ColumnStats(orderKey, 2L, 11L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day2), new ColumnStats(orderKey, 2L, null), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, null), new ColumnStats(orderKey, 2L, 11L), new ColumnStats(orderStatus, "aaa", "abc")))).build();
    long transactionId = shardManager.beginTransaction();
    shardManager.commitShards(transactionId, tableInfo.getTableId(), COLUMNS, shards, Optional.empty(), 0);
    Set<ShardMetadata> shardMetadatas = shardManager.getNodeShards("node1");
    Long temporalColumnId = metadataDao.getTemporalColumnId(tableInfo.getTableId());
    TableColumn temporalColumn = metadataDao.getTableColumn(tableInfo.getTableId(), temporalColumnId);
    Set<ShardIndexInfo> actual = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, false));
    List<ShardIndexInfo> expected = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.empty());
    assertEquals(actual, expected);
    List<TableColumn> sortColumns = metadataDao.listSortColumns(tableInfo.getTableId());
    Set<ShardIndexInfo> actualSortRange = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, true));
    List<ShardIndexInfo> expectedSortRange = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.of(sortColumns));
    assertEquals(actualSortRange, expectedSortRange);
}