Examples with TableStatistics com.facebook.presto.spi.statistics.TableStatistics used on opensource projects

Example 16 with TableStatistics

use of com.facebook.presto.spi.statistics.TableStatistics in project presto by prestodb.

the class ConnectorFilterStatsCalculatorService method filterStats.

@Override
public TableStatistics filterStats(TableStatistics tableStatistics, RowExpression predicate, ConnectorSession session, Map<ColumnHandle, String> columnNames, Map<String, Type> columnTypes) {
    PlanNodeStatsEstimate tableStats = toPlanNodeStats(tableStatistics, columnNames, columnTypes);
    PlanNodeStatsEstimate filteredStats = filterStatsCalculator.filterStats(tableStats, predicate, session);
    if (filteredStats.isOutputRowCountUnknown()) {
        filteredStats = tableStats.mapOutputRowCount(sourceRowCount -> tableStats.getOutputRowCount() * UNKNOWN_FILTER_COEFFICIENT);
    }
    TableStatistics filteredStatistics = toTableStatistics(filteredStats, ImmutableBiMap.copyOf(columnNames).inverse());
    // Fill in the totalSize after filter, estimated proportional to the rowCount after versus before filter.
    TableStatistics.Builder filteredStatsWithSize = TableStatistics.builder();
    filteredStatsWithSize.setRowCount(filteredStatistics.getRowCount());
    filteredStatistics.getColumnStatistics().forEach(filteredStatsWithSize::setColumnStatistics);
    // If the rowCount before or after filter is zero, totalSize will also be zero
    if (!tableStatistics.getRowCount().isUnknown() && tableStatistics.getRowCount().getValue() == 0 || !filteredStatistics.getRowCount().isUnknown() && filteredStatistics.getRowCount().getValue() == 0) {
        filteredStatsWithSize.setTotalSize(Estimate.of(0));
    } else if (!tableStatistics.getTotalSize().isUnknown() && !filteredStatistics.getRowCount().isUnknown() && !tableStatistics.getRowCount().isUnknown()) {
        double totalSizeAfterFilter = filteredStatistics.getRowCount().getValue() / tableStatistics.getRowCount().getValue() * tableStatistics.getTotalSize().getValue();
        filteredStatsWithSize.setTotalSize(Estimate.of(totalSizeAfterFilter));
    }
    return filteredStatsWithSize.build();
}

Example 17 with TableStatistics

use of com.facebook.presto.spi.statistics.TableStatistics in project presto by prestodb.

the class AbstractTestHiveClient method assertTableStatsComputed.

private void assertTableStatsComputed(SchemaTableName tableName, Set<String> expectedColumnStatsColumns) {
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
        List<ColumnHandle> allColumnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values());
        TableStatistics tableStatistics = metadata.getTableStatistics(session, tableHandle, Optional.empty(), allColumnHandles, Constraint.alwaysTrue());
        assertFalse(tableStatistics.getRowCount().isUnknown(), "row count is unknown");
        Map<String, ColumnStatistics> columnsStatistics = tableStatistics.getColumnStatistics().entrySet().stream().collect(toImmutableMap(entry -> ((HiveColumnHandle) entry.getKey()).getName(), Map.Entry::getValue));
        assertEquals(columnsStatistics.keySet(), expectedColumnStatsColumns, "columns with statistics");
        Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle);
        columnsStatistics.forEach((columnName, columnStatistics) -> {
            ColumnHandle columnHandle = columnHandles.get(columnName);
            Type columnType = metadata.getColumnMetadata(session, tableHandle, columnHandle).getType();
            assertFalse(columnStatistics.getNullsFraction().isUnknown(), "unknown nulls fraction for " + columnName);
            assertFalse(columnStatistics.getDistinctValuesCount().isUnknown(), "unknown distinct values count for " + columnName);
            if (isVarcharType(columnType)) {
                assertFalse(columnStatistics.getDataSize().isUnknown(), "unknown data size for " + columnName);
            } else {
                assertTrue(columnStatistics.getDataSize().isUnknown(), "unknown data size for" + columnName);
            }
        });
    }
}

Example 18 with TableStatistics

use of com.facebook.presto.spi.statistics.TableStatistics in project presto by prestodb.

the class TpcdsTableStatisticsFactory method toTableStatistics.

private TableStatistics toTableStatistics(List<ColumnHandle> columnHandles, TableStatisticsData statisticsData) {
    long rowCount = statisticsData.getRowCount();
    TableStatistics.Builder tableStatistics = TableStatistics.builder().setRowCount(Estimate.of(rowCount));
    if (rowCount > 0) {
        Map<String, ColumnStatisticsData> columnsData = statisticsData.getColumns();
        for (ColumnHandle handle : columnHandles) {
            TpcdsColumnHandle columnHandle = (TpcdsColumnHandle) handle;
            tableStatistics.setColumnStatistics(columnHandle, toColumnStatistics(columnsData.get(columnHandle.getColumnName()), columnHandle.getType(), rowCount));
        }
    }
    return tableStatistics.build();
}

Example 19 with TableStatistics

use of com.facebook.presto.spi.statistics.TableStatistics in project presto by prestodb.

the class TestTpcdsMetadataStatistics method testTableStatisticsSerialization.

@Test
public void testTableStatisticsSerialization() {
    SchemaTableName schemaTableName = new SchemaTableName("sf1", Table.WEB_SITE.getName());
    ConnectorTableHandle tableHandle = metadata.getTableHandle(session, schemaTableName);
    List<ColumnHandle> columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values());
    TableStatistics tableStatistics = metadata.getTableStatistics(session, tableHandle, Optional.empty(), columnHandles, alwaysTrue());
    Entry<ColumnHandle, ColumnStatistics> entry = tableStatistics.getColumnStatistics().entrySet().iterator().next();
    TableStatistics expectedTableStatictics = tableStatistics.builder().setRowCount(tableStatistics.getRowCount()).setColumnStatistics(entry.getKey(), entry.getValue()).build();
    JsonCodec<TableStatistics> codec = JsonCodec.jsonCodec(TableStatistics.class);
    String json = codec.toJson(expectedTableStatictics);
    assertEquals(json, "{\n" + "  \"rowCount\" : {\n" + "    \"value\" : 30.0\n" + "  },\n" + "  \"totalSize\" : {\n" + "    \"value\" : \"NaN\"\n" + "  },\n" + "  \"columnStatistics\" : {\n" + "    \"tpcds:web_site_sk\" : {\n" + "      \"nullsFraction\" : {\n" + "        \"value\" : 0.0\n" + "      },\n" + "      \"distinctValuesCount\" : {\n" + "        \"value\" : 30.0\n" + "      },\n" + "      \"dataSize\" : {\n" + "        \"value\" : \"NaN\"\n" + "      },\n" + "      \"range\" : {\n" + "        \"min\" : 1.0,\n" + "        \"max\" : 30.0\n" + "      }\n" + "    }\n" + "  }\n" + "}");
}

Example 20 with TableStatistics

use of com.facebook.presto.spi.statistics.TableStatistics in project presto by prestodb.

the class TestTpchMetadata method testNoTableStats.

private void testNoTableStats(String schema, TpchTable<?> table) {
    TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName(schema, table.getTableName()));
    List<ColumnHandle> columnHandles = ImmutableList.copyOf(tpchMetadata.getColumnHandles(session, tableHandle).values());
    TableStatistics tableStatistics = tpchMetadata.getTableStatistics(session, tableHandle, Optional.empty(), columnHandles, alwaysTrue());
    assertTrue(tableStatistics.getRowCount().isUnknown());
}