Examples with ColumnStats com.facebook.presto.raptor.metadata.ColumnStats used on opensource projects

Example 1 with ColumnStats

use of com.facebook.presto.raptor.metadata.ColumnStats in project presto by prestodb.

the class ShardStats method indexLong.

private static ColumnStats indexLong(Type type, OrcRecordReader reader, int columnIndex, long columnId) throws IOException {
    boolean minSet = false;
    boolean maxSet = false;
    long min = 0;
    long max = 0;
    while (true) {
        int batchSize = reader.nextBatch();
        if (batchSize <= 0) {
            break;
        }
        Block block = reader.readBlock(type, columnIndex);
        for (int i = 0; i < batchSize; i++) {
            if (block.isNull(i)) {
                continue;
            }
            long value = type.getLong(block, i);
            if (!minSet || (value < min)) {
                minSet = true;
                min = value;
            }
            if (!maxSet || (value > max)) {
                maxSet = true;
                max = value;
            }
        }
    }
    return new ColumnStats(columnId, minSet ? min : null, maxSet ? max : null);
}

Example 2 with ColumnStats

use of com.facebook.presto.raptor.metadata.ColumnStats in project presto by prestodb.

the class ShardStats method indexBoolean.

private static ColumnStats indexBoolean(Type type, OrcRecordReader reader, int columnIndex, long columnId) throws IOException {
    boolean minSet = false;
    boolean maxSet = false;
    boolean min = false;
    boolean max = false;
    while (true) {
        int batchSize = reader.nextBatch();
        if (batchSize <= 0) {
            break;
        }
        Block block = reader.readBlock(type, columnIndex);
        for (int i = 0; i < batchSize; i++) {
            if (block.isNull(i)) {
                continue;
            }
            boolean value = type.getBoolean(block, i);
            if (!minSet || Boolean.compare(value, min) < 0) {
                minSet = true;
                min = value;
            }
            if (!maxSet || Boolean.compare(value, max) > 0) {
                maxSet = true;
                max = value;
            }
        }
    }
    return new ColumnStats(columnId, minSet ? min : null, maxSet ? max : null);
}

Example 3 with ColumnStats

use of com.facebook.presto.raptor.metadata.ColumnStats in project presto by prestodb.

the class ShardStats method indexDouble.

private static ColumnStats indexDouble(Type type, OrcRecordReader reader, int columnIndex, long columnId) throws IOException {
    boolean minSet = false;
    boolean maxSet = false;
    double min = 0;
    double max = 0;
    while (true) {
        int batchSize = reader.nextBatch();
        if (batchSize <= 0) {
            break;
        }
        Block block = reader.readBlock(type, columnIndex);
        for (int i = 0; i < batchSize; i++) {
            if (block.isNull(i)) {
                continue;
            }
            double value = type.getDouble(block, i);
            if (isNaN(value)) {
                continue;
            }
            if (value == -0.0) {
                value = 0.0;
            }
            if (!minSet || (value < min)) {
                minSet = true;
                min = value;
            }
            if (!maxSet || (value > max)) {
                maxSet = true;
                max = value;
            }
        }
    }
    if (isInfinite(min)) {
        minSet = false;
    }
    if (isInfinite(max)) {
        maxSet = false;
    }
    return new ColumnStats(columnId, minSet ? min : null, maxSet ? max : null);
}

Example 4 with ColumnStats

use of com.facebook.presto.raptor.metadata.ColumnStats in project presto by prestodb.

the class ShardStats method indexString.

private static ColumnStats indexString(Type type, OrcRecordReader reader, int columnIndex, long columnId) throws IOException {
    boolean minSet = false;
    boolean maxSet = false;
    Slice min = null;
    Slice max = null;
    while (true) {
        int batchSize = reader.nextBatch();
        if (batchSize <= 0) {
            break;
        }
        Block block = reader.readBlock(type, columnIndex);
        for (int i = 0; i < batchSize; i++) {
            if (block.isNull(i)) {
                continue;
            }
            Slice slice = type.getSlice(block, i);
            slice = truncateIndexValue(slice);
            if (!minSet || (slice.compareTo(min) < 0)) {
                minSet = true;
                min = slice;
            }
            if (!maxSet || (slice.compareTo(max) > 0)) {
                maxSet = true;
                max = slice;
            }
        }
    }
    return new ColumnStats(columnId, minSet ? min.toStringUtf8() : null, maxSet ? max.toStringUtf8() : null);
}

Example 5 with ColumnStats

use of com.facebook.presto.raptor.metadata.ColumnStats in project presto by prestodb.

the class TestShardOrganizerUtil method testGetOrganizationEligibleShards.

@Test
public void testGetOrganizationEligibleShards() throws Exception {
    int day1 = 1111;
    int day2 = 2222;
    SchemaTableName tableName = new SchemaTableName("default", "test");
    metadata.createTable(SESSION, tableMetadataBuilder(tableName).column("orderkey", BIGINT).column("orderdate", DATE).column("orderstatus", createVarcharType(3)).property("ordering", ImmutableList.of("orderstatus", "orderkey")).property("temporal_column", "orderdate").build());
    Table tableInfo = metadataDao.getTableInformation(tableName.getSchemaName(), tableName.getTableName());
    List<TableColumn> tableColumns = metadataDao.listTableColumns(tableInfo.getTableId());
    Map<String, TableColumn> tableColumnMap = Maps.uniqueIndex(tableColumns, TableColumn::getColumnName);
    long orderDate = tableColumnMap.get("orderdate").getColumnId();
    long orderKey = tableColumnMap.get("orderkey").getColumnId();
    long orderStatus = tableColumnMap.get("orderstatus").getColumnId();
    List<ShardInfo> shards = ImmutableList.<ShardInfo>builder().add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day1 + 10), new ColumnStats(orderKey, 13L, 14L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day2, day2 + 100), new ColumnStats(orderKey, 2L, 20L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day2), new ColumnStats(orderKey, 2L, 11L), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, day2), new ColumnStats(orderKey, 2L, null), new ColumnStats(orderStatus, "aaa", "abc")))).add(shardInfo(UUID.randomUUID(), "node1", ImmutableList.of(new ColumnStats(orderDate, day1, null), new ColumnStats(orderKey, 2L, 11L), new ColumnStats(orderStatus, "aaa", "abc")))).build();
    long transactionId = shardManager.beginTransaction();
    shardManager.commitShards(transactionId, tableInfo.getTableId(), COLUMNS, shards, Optional.empty(), 0);
    Set<ShardMetadata> shardMetadatas = shardManager.getNodeShards("node1");
    Long temporalColumnId = metadataDao.getTemporalColumnId(tableInfo.getTableId());
    TableColumn temporalColumn = metadataDao.getTableColumn(tableInfo.getTableId(), temporalColumnId);
    Set<ShardIndexInfo> actual = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, false));
    List<ShardIndexInfo> expected = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.empty());
    assertEquals(actual, expected);
    List<TableColumn> sortColumns = metadataDao.listSortColumns(tableInfo.getTableId());
    Set<ShardIndexInfo> actualSortRange = ImmutableSet.copyOf(getOrganizationEligibleShards(dbi, metadataDao, tableInfo, shardMetadatas, true));
    List<ShardIndexInfo> expectedSortRange = getShardIndexInfo(tableInfo, shards, temporalColumn, Optional.of(sortColumns));
    assertEquals(actualSortRange, expectedSortRange);
}