Examples with ColumnStatistics org.apache.ignite.internal.processors.query.stat.ColumnStatistics used on opensource projects

Example 1 with ColumnStatistics

use of org.apache.ignite.internal.processors.query.stat.ColumnStatistics in project ignite by apache.

the class ColumnLocalDataViewSupplier method columnLocalStatisticsViewSupplier.

/**
 * Statistics column local node data view filterable supplier.
 *
 * @param filter Filter.
 * @return Iterable with statistics column local node data views.
 */
public Iterable<StatisticsColumnLocalDataView> columnLocalStatisticsViewSupplier(Map<String, Object> filter) {
    String type = (String) filter.get(StatisticsColumnPartitionDataViewWalker.TYPE_FILTER);
    if (type != null && !StatisticsColumnConfigurationView.TABLE_TYPE.equalsIgnoreCase(type))
        return Collections.emptyList();
    String schema = (String) filter.get(StatisticsColumnLocalDataViewWalker.SCHEMA_FILTER);
    String name = (String) filter.get(StatisticsColumnLocalDataViewWalker.NAME_FILTER);
    String column = (String) filter.get(StatisticsColumnPartitionDataViewWalker.COLUMN_FILTER);
    Map<StatisticsKey, ObjectStatisticsImpl> locStatsMap;
    if (!F.isEmpty(schema) && !F.isEmpty(name)) {
        StatisticsKey key = new StatisticsKey(schema, name);
        ObjectStatisticsImpl objLocStat = repo.getLocalStatistics(key);
        if (objLocStat == null)
            return Collections.emptyList();
        locStatsMap = Collections.singletonMap(key, objLocStat);
    } else
        locStatsMap = repo.localStatisticsMap().entrySet().stream().filter(e -> F.isEmpty(schema) || schema.equals(e.getKey().schema())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    List<StatisticsColumnLocalDataView> res = new ArrayList<>();
    for (Map.Entry<StatisticsKey, ObjectStatisticsImpl> localStatsEntry : locStatsMap.entrySet()) {
        StatisticsKey key = localStatsEntry.getKey();
        ObjectStatisticsImpl stat = localStatsEntry.getValue();
        if (column == null) {
            for (Map.Entry<String, ColumnStatistics> colStat : localStatsEntry.getValue().columnsStatistics().entrySet()) {
                StatisticsColumnLocalDataView colStatView = new StatisticsColumnLocalDataView(key, colStat.getKey(), stat);
                res.add(colStatView);
            }
        } else {
            ColumnStatistics colStat = localStatsEntry.getValue().columnStatistics(column);
            if (colStat != null) {
                StatisticsColumnLocalDataView colStatView = new StatisticsColumnLocalDataView(key, column, stat);
                res.add(colStatView);
            }
        }
    }
    return res;
}

Example 2 with ColumnStatistics

use of org.apache.ignite.internal.processors.query.stat.ColumnStatistics in project ignite by apache.

the class H2IndexCostedBase method rowCost.

/**
 * Row cost calculation.
 *
 * @param ses Session.
 * @param filter Table filter.
 * @param masks Masks array.
 * @param rowCount Total rows count.
 * @param locTblStats Local table statistics.
 * @return Row cost.
 */
private long rowCost(Session ses, TableFilter filter, int[] masks, long rowCount, ObjectStatisticsImpl locTblStats) {
    double totalCardinality = 0;
    long rowsCost = rowCount;
    if (masks != null) {
        int i = 0, len = columns.length;
        while (i < len) {
            Column column = columns[i++];
            ColumnStatistics colStats = getColumnStatistics(locTblStats, column);
            int index = column.getColumnId();
            int mask = masks[index];
            if (isByteFlag(mask, IndexCondition.EQUALITY)) {
                if (i == len && getIndexType().isUnique()) {
                    rowsCost = 3;
                    break;
                }
                // Estimate by is null
                Value equalValue = getEqualValue(ses, column, filter);
                Boolean equalNull = (equalValue == null) ? null : equalValue.getType() == Value.NULL;
                rowCount = getColumnSize(colStats, rowCount, equalNull);
                if (colStats != null && equalNull == Boolean.TRUE) {
                    rowsCost = Math.min(5 + Math.max(Math.round(rowsCost * nulls(colStats)), 1), rowsCost - (i > 0 ? 1 : 0));
                    continue;
                }
                if (colStats != null && equalNull == Boolean.FALSE)
                    rowsCost = Math.max(Math.round(rowsCost * (1 - nulls(colStats))), 1);
                long distinctRows;
                if (colStats == null) {
                    double cardinality = (double) column.getSelectivity() / 100;
                    totalCardinality = 1 - (1 - totalCardinality) * (1 - cardinality);
                    distinctRows = Math.round((double) rowCount * totalCardinality);
                } else {
                    double cardinality;
                    long nonNulls = colStats.total() - colStats.nulls();
                    if (nonNulls == 0)
                        cardinality = 1;
                    else
                        cardinality = (double) colStats.distinct() / nonNulls;
                    totalCardinality = 1 - (1 - totalCardinality) * (1 - cardinality);
                    distinctRows = Math.round(rowCount * totalCardinality);
                }
                if (distinctRows <= 0)
                    distinctRows = 1;
                rowsCost = Math.min(5 + Math.max(rowsCost / distinctRows, 1), rowsCost - (i > 0 ? 1 : 0));
            } else if (isByteFlag(mask, IndexCondition.RANGE) || isByteFlag(mask, IndexCondition.START) || isByteFlag(mask, IndexCondition.END)) {
                Value min = getStartValue(ses, column, filter);
                Value max = getEndValue(ses, column, filter);
                int percent = estimatePercent(colStats, min, max);
                rowsCost = Math.min(5 + rowsCost * percent / 100, rowsCost - (i > 0 ? 1 : 0));
                break;
            } else if (isNullFilter(ses, column, filter)) {
                if (colStats != null)
                    rowsCost = Math.min(5 + colStats.nulls(), rowsCost - (i > 0 ? 1 : 0));
                break;
            } else if (isNotNullFilter(ses, column, filter)) {
                if (colStats != null)
                    rowsCost = Math.min(5 + colStats.total() - colStats.nulls(), rowsCost - (i > 0 ? 1 : 0));
                break;
            } else
                break;
        }
    }
    return rowsCost;
}

Example 3 with ColumnStatistics

use of org.apache.ignite.internal.processors.query.stat.ColumnStatistics in project ignite by apache.

the class ColumnPartitionDataViewSupplier method columnPartitionStatisticsViewSupplier.

/**
 * Statistics column partition data view filterable supplier.
 *
 * @param filter Filter.
 * @return Iterable with statistics column partition data views.
 */
public Iterable<StatisticsColumnPartitionDataView> columnPartitionStatisticsViewSupplier(Map<String, Object> filter) {
    String type = (String) filter.get(StatisticsColumnPartitionDataViewWalker.TYPE_FILTER);
    if (type != null && !StatisticsColumnConfigurationView.TABLE_TYPE.equalsIgnoreCase(type))
        return Collections.emptyList();
    String schema = (String) filter.get(StatisticsColumnPartitionDataViewWalker.SCHEMA_FILTER);
    String name = (String) filter.get(StatisticsColumnPartitionDataViewWalker.NAME_FILTER);
    Integer partId = (Integer) filter.get(StatisticsColumnPartitionDataViewWalker.PARTITION_FILTER);
    String column = (String) filter.get(StatisticsColumnPartitionDataViewWalker.COLUMN_FILTER);
    Map<StatisticsKey, Collection<ObjectPartitionStatisticsImpl>> partsStatsMap;
    if (!F.isEmpty(schema) && !F.isEmpty(name)) {
        StatisticsKey key = new StatisticsKey(schema, name);
        Collection<ObjectPartitionStatisticsImpl> keyStat;
        if (partId == null)
            keyStat = store.getLocalPartitionsStatistics(key);
        else {
            ObjectPartitionStatisticsImpl partStat = store.getLocalPartitionStatistics(key, partId);
            keyStat = (partStat == null) ? Collections.emptyList() : Collections.singletonList(partStat);
        }
        partsStatsMap = Collections.singletonMap(key, keyStat);
    } else
        partsStatsMap = store.getAllLocalPartitionsStatistics(schema);
    List<StatisticsColumnPartitionDataView> res = new ArrayList<>();
    for (Map.Entry<StatisticsKey, Collection<ObjectPartitionStatisticsImpl>> partsStatsEntry : partsStatsMap.entrySet()) {
        StatisticsKey key = partsStatsEntry.getKey();
        for (ObjectPartitionStatisticsImpl partStat : partsStatsEntry.getValue()) {
            if (column == null) {
                for (Map.Entry<String, ColumnStatistics> colStatEntry : partStat.columnsStatistics().entrySet()) res.add(new StatisticsColumnPartitionDataView(key, colStatEntry.getKey(), partStat));
            } else {
                ColumnStatistics colStat = partStat.columnStatistics(column);
                if (colStat != null)
                    res.add(new StatisticsColumnPartitionDataView(key, column, partStat));
            }
        }
    }
    return res;
}

Example 4 with ColumnStatistics

use of org.apache.ignite.internal.processors.query.stat.ColumnStatistics in project ignite by apache.

the class GatherPartitionStatistics method fixExisting.

/**
 * Fix existing partition statistics, update repo and return resulting partition statistics.
 *
 * @param partStat Partition statistics to fix.
 * @param colsToRemove Columns to remove.
 * @return New "fixed" partition statistics or existing, if colsToRemove is empty.
 */
private ObjectPartitionStatisticsImpl fixExisting(ObjectPartitionStatisticsImpl partStat, Set<String> colsToRemove) {
    if (log.isDebugEnabled())
        log.debug("Existing parititon statistics fit to configuration requirements. " + "Skipping recollection for " + gathCtx.configuration().key() + "[" + partId + "].");
    ObjectPartitionStatisticsImpl res;
    if (F.isEmpty(colsToRemove))
        // No changes - no need to write existing parition back.
        res = partStat;
    else {
        Map<String, ColumnStatistics> allCols = new HashMap<>(partStat.columnsStatistics());
        for (String col : colsToRemove) allCols.remove(col);
        res = new ObjectPartitionStatisticsImpl(partStat.partId(), getRowCount(allCols), partStat.updCnt(), allCols);
        assert !allCols.isEmpty() : "No columns left after fixing existing partition statistics.";
        statRepo.replaceLocalPartitionStatistics(gathCtx.configuration().key(), res);
    }
    return res;
}

Example 5 with ColumnStatistics

use of org.apache.ignite.internal.processors.query.stat.ColumnStatistics in project ignite by apache.

the class GatherPartitionStatistics method recollectPartition.

/**
 * Collect some statistics, fix existing in repo and return resulting partition statistics.
 *
 * @param cctx Cache context to get partition from.
 * @param partStat Existing partition statistics to fix or use as a base.
 * @param colsToCollect Columns to collect.
 * @param colsToRemove Columns to remove.
 * @return New partition statistics.
 */
private ObjectPartitionStatisticsImpl recollectPartition(GridCacheContext<?, ?> cctx, ObjectPartitionStatisticsImpl partStat, Map<String, StatisticsColumnConfiguration> colsToCollect, Set<String> colsToRemove) {
    CacheGroupContext grp = cctx.group();
    GridDhtPartitionTopology top = grp.topology();
    AffinityTopologyVersion topVer = top.readyTopologyVersion();
    GridDhtLocalPartition locPart = top.localPartition(partId, topVer, false);
    if (locPart == null)
        throw new GatherStatisticCancelException();
    boolean reserved = locPart.reserve();
    GridH2Table tbl = gathCtx.table();
    ObjectPartitionStatisticsImpl res = null;
    try {
        if (!reserved || (locPart.state() != OWNING)) {
            if (log.isDebugEnabled()) {
                log.debug("Partition not owning. Need to retry [part=" + partId + ", tbl=" + tbl.identifier() + ']');
            }
            throw new GatherStatisticCancelException();
        }
        Column[] cols = IgniteStatisticsHelper.filterColumns(tbl.getColumns(), colsToCollect.keySet());
        ColumnStatisticsCollector[] collectors = new ColumnStatisticsCollector[cols.length];
        for (int i = 0; i < cols.length; ++i) {
            long colCfgVer = colsToCollect.get(cols[i].getName()).version();
            collectors[i] = new ColumnStatisticsCollector(cols[i], tbl::compareTypeSafe, colCfgVer);
        }
        GridH2RowDescriptor rowDesc = tbl.rowDescriptor();
        GridQueryTypeDescriptor typeDesc = rowDesc.type();
        try {
            int checkInt = CANCELLED_CHECK_INTERVAL;
            if (log.isDebugEnabled()) {
                log.debug("Start partition scan [part=" + partId + ", tbl=" + gathCtx.table().identifier() + ']');
            }
            for (CacheDataRow row : grp.offheap().cachePartitionIterator(gathCtx.table().cacheId(), partId, null, false)) {
                if (--checkInt == 0) {
                    if (gathCtx.future().isCancelled())
                        throw new GatherStatisticCancelException();
                    checkInt = CANCELLED_CHECK_INTERVAL;
                }
                if (!typeDesc.matchType(row.value()) || wasExpired(row))
                    continue;
                H2Row h2row = rowDesc.createRow(row);
                for (ColumnStatisticsCollector colStat : collectors) colStat.add(h2row.getValue(colStat.col().getColumnId()));
            }
        } catch (IgniteCheckedException e) {
            log.warning(String.format("Unable to collect partition level statistics by %s.%s:%d due to %s", tbl.identifier().schema(), tbl.identifier().table(), partId, e.getMessage()));
            throw new IgniteException("Unable to collect partition level statistics", e);
        }
        Map<String, ColumnStatistics> colStats = Arrays.stream(collectors).collect(Collectors.toMap(csc -> csc.col().getName(), ColumnStatisticsCollector::finish));
        // Add existing to full replace existing statistics with new one.
        if (partStat != null) {
            for (Map.Entry<String, ColumnStatistics> oldColStat : partStat.columnsStatistics().entrySet()) {
                if (!colsToRemove.contains(oldColStat.getKey()))
                    colStats.putIfAbsent(oldColStat.getKey(), oldColStat.getValue());
            }
        }
        res = new ObjectPartitionStatisticsImpl(partId, getRowCount(colStats), locPart.updateCounter(), colStats);
    } finally {
        if (reserved)
            locPart.release();
    }
    statRepo.replaceLocalPartitionStatistics(gathCtx.configuration().key(), res);
    if (gathCtx.configuration().columns().size() == colsToCollect.size())
        statRepo.refreshObsolescence(gathCtx.configuration().key(), partId);
    return res;
}