Examples with StatisticsColumnConfiguration org.apache.ignite.internal.processors.query.stat.config.StatisticsColumnConfiguration used on opensource projects

Example 6 with StatisticsColumnConfiguration

use of org.apache.ignite.internal.processors.query.stat.config.StatisticsColumnConfiguration in project ignite by apache.

the class StatisticsAbstractTest method makeStatistics.

/**
 * Collect or refresh statistics.
 *
 * @param collect If {@code true} - collect new statistics, if {@code false} - update existing.
 * @param targets
 */
private void makeStatistics(boolean collect, StatisticsTarget... targets) {
    try {
        Map<StatisticsTarget, Long> expectedVersion = new HashMap<>();
        IgniteStatisticsManagerImpl statMgr = statisticsMgr(0);
        for (StatisticsTarget target : targets) {
            StatisticsObjectConfiguration currCfg = statMgr.statisticConfiguration().config(target.key());
            Predicate<StatisticsColumnConfiguration> pred;
            if (F.isEmpty(target.columns()))
                pred = c -> true;
            else {
                Set<String> cols = Arrays.stream(target.columns()).collect(Collectors.toSet());
                pred = c -> cols.contains(c.name());
            }
            Long expVer = (currCfg == null) ? 1L : currCfg.columnsAll().values().stream().filter(pred).mapToLong(StatisticsColumnConfiguration::version).min().orElse(0L) + 1;
            expectedVersion.put(target, expVer);
        }
        if (collect)
            statisticsMgr(0).collectStatistics(buildDefaultConfigurations(targets));
        else
            statisticsMgr(0).refreshStatistics(targets);
        awaitStatistics(TIMEOUT, expectedVersion);
    } catch (Exception ex) {
        throw new IgniteException(ex);
    }
}

Example 7 with StatisticsColumnConfiguration

use of org.apache.ignite.internal.processors.query.stat.config.StatisticsColumnConfiguration in project ignite by apache.

the class IgniteStatisticsHelper method aggregateLocalStatistics.

/**
 * Aggregate partition level statistics to local level one or local statistics to global one.
 *
 * @param tbl Table to aggregate statistics by.
 * @param cfg Statistics object configuration.
 * @param stats Collection of partition level or local level statistics to aggregate.
 * @param log Logger.
 * @return Local level statistics.
 */
public static ObjectStatisticsImpl aggregateLocalStatistics(GridH2Table tbl, StatisticsObjectConfiguration cfg, Collection<? extends ObjectStatisticsImpl> stats, IgniteLogger log) {
    assert !stats.isEmpty();
    Column[] selectedCols = filterColumns(tbl.getColumns(), cfg.columns().keySet());
    Map<Column, List<ColumnStatistics>> colPartStats = new HashMap<>(selectedCols.length);
    long rowCnt = 0;
    for (Column col : selectedCols) colPartStats.put(col, new ArrayList<>());
    for (ObjectStatisticsImpl partStat : stats) {
        for (Column col : selectedCols) {
            ColumnStatistics colPartStat = partStat.columnStatistics(col.getName());
            if (colPartStat != null)
                colPartStats.get(col).add(colPartStat);
        }
        rowCnt += partStat.rowCount();
    }
    Map<String, ColumnStatistics> colStats = new HashMap<>(selectedCols.length);
    for (Column col : selectedCols) {
        StatisticsColumnConfiguration colCfg = cfg.columns().get(col.getName());
        ColumnStatistics stat = ColumnStatisticsCollector.aggregate(tbl::compareTypeSafe, colPartStats.get(col), colCfg.overrides());
        if (log.isDebugEnabled())
            log.debug("Aggregate column statistic done [col=" + col.getName() + ", stat=" + stat + ']');
        colStats.put(col.getName(), stat);
    }
    rowCnt = calculateRowCount(cfg, rowCnt);
    return new ObjectStatisticsImpl(rowCnt, colStats);
}

Example 8 with StatisticsColumnConfiguration

use of org.apache.ignite.internal.processors.query.stat.config.StatisticsColumnConfiguration in project ignite by apache.

the class IgniteStatisticsHelper method buildDefaultConfigurations.

/**
 * Build object configurations array with all default parameters from specified targets.
 *
 * @param targets Targets to build configurations from.
 * @return StatisticsObjectConfiguration array.
 */
public static StatisticsObjectConfiguration[] buildDefaultConfigurations(StatisticsTarget... targets) {
    StatisticsObjectConfiguration[] res = Arrays.stream(targets).map(t -> {
        List<StatisticsColumnConfiguration> colCfgs;
        if (t.columns() == null)
            colCfgs = Collections.emptyList();
        else
            colCfgs = Arrays.stream(t.columns()).map(name -> new StatisticsColumnConfiguration(name, null)).collect(Collectors.toList());
        return new StatisticsObjectConfiguration(t.key(), colCfgs, StatisticsObjectConfiguration.DEFAULT_OBSOLESCENCE_MAX_PERCENT);
    }).toArray(StatisticsObjectConfiguration[]::new);
    return res;
}

Example 9 with StatisticsColumnConfiguration

use of org.apache.ignite.internal.processors.query.stat.config.StatisticsColumnConfiguration in project ignite by apache.

the class IgniteStatisticsHelper method calculateRowCount.

/**
 * Calculate effective row count. If there are some overrides in statistics configuration - maximum value will be
 * choosen. If not - will return actualRowCount.
 *
 * @param cfg Statistics configuration to dig overrides row count from.
 * @param actualRowCount Actual row count.
 * @return Effective row count.
 */
public static long calculateRowCount(StatisticsObjectConfiguration cfg, long actualRowCount) {
    long overridedRowCnt = -1;
    for (StatisticsColumnConfiguration ccfg : cfg.columns().values()) {
        if (ccfg.overrides() != null && ccfg.overrides().total() != null) {
            Long colRowCnt = ccfg.overrides().total();
            overridedRowCnt = Math.max(overridedRowCnt, colRowCnt);
        }
    }
    return (overridedRowCnt == -1) ? actualRowCount : overridedRowCnt;
}

Example 10 with StatisticsColumnConfiguration

use of org.apache.ignite.internal.processors.query.stat.config.StatisticsColumnConfiguration in project ignite by apache.

the class GatherPartitionStatistics method recollectPartition.

/**
 * Collect some statistics, fix existing in repo and return resulting partition statistics.
 *
 * @param cctx Cache context to get partition from.
 * @param partStat Existing partition statistics to fix or use as a base.
 * @param colsToCollect Columns to collect.
 * @param colsToRemove Columns to remove.
 * @return New partition statistics.
 */
private ObjectPartitionStatisticsImpl recollectPartition(GridCacheContext<?, ?> cctx, ObjectPartitionStatisticsImpl partStat, Map<String, StatisticsColumnConfiguration> colsToCollect, Set<String> colsToRemove) {
    CacheGroupContext grp = cctx.group();
    GridDhtPartitionTopology top = grp.topology();
    AffinityTopologyVersion topVer = top.readyTopologyVersion();
    GridDhtLocalPartition locPart = top.localPartition(partId, topVer, false);
    if (locPart == null)
        throw new GatherStatisticCancelException();
    boolean reserved = locPart.reserve();
    GridH2Table tbl = gathCtx.table();
    ObjectPartitionStatisticsImpl res = null;
    try {
        if (!reserved || (locPart.state() != OWNING)) {
            if (log.isDebugEnabled()) {
                log.debug("Partition not owning. Need to retry [part=" + partId + ", tbl=" + tbl.identifier() + ']');
            }
            throw new GatherStatisticCancelException();
        }
        Column[] cols = IgniteStatisticsHelper.filterColumns(tbl.getColumns(), colsToCollect.keySet());
        ColumnStatisticsCollector[] collectors = new ColumnStatisticsCollector[cols.length];
        for (int i = 0; i < cols.length; ++i) {
            long colCfgVer = colsToCollect.get(cols[i].getName()).version();
            collectors[i] = new ColumnStatisticsCollector(cols[i], tbl::compareTypeSafe, colCfgVer);
        }
        GridH2RowDescriptor rowDesc = tbl.rowDescriptor();
        GridQueryTypeDescriptor typeDesc = rowDesc.type();
        try {
            int checkInt = CANCELLED_CHECK_INTERVAL;
            if (log.isDebugEnabled()) {
                log.debug("Start partition scan [part=" + partId + ", tbl=" + gathCtx.table().identifier() + ']');
            }
            for (CacheDataRow row : grp.offheap().cachePartitionIterator(gathCtx.table().cacheId(), partId, null, false)) {
                if (--checkInt == 0) {
                    if (gathCtx.future().isCancelled())
                        throw new GatherStatisticCancelException();
                    checkInt = CANCELLED_CHECK_INTERVAL;
                }
                if (!typeDesc.matchType(row.value()) || wasExpired(row))
                    continue;
                H2Row h2row = rowDesc.createRow(row);
                for (ColumnStatisticsCollector colStat : collectors) colStat.add(h2row.getValue(colStat.col().getColumnId()));
            }
        } catch (IgniteCheckedException e) {
            log.warning(String.format("Unable to collect partition level statistics by %s.%s:%d due to %s", tbl.identifier().schema(), tbl.identifier().table(), partId, e.getMessage()));
            throw new IgniteException("Unable to collect partition level statistics", e);
        }
        Map<String, ColumnStatistics> colStats = Arrays.stream(collectors).collect(Collectors.toMap(csc -> csc.col().getName(), ColumnStatisticsCollector::finish));
        // Add existing to full replace existing statistics with new one.
        if (partStat != null) {
            for (Map.Entry<String, ColumnStatistics> oldColStat : partStat.columnsStatistics().entrySet()) {
                if (!colsToRemove.contains(oldColStat.getKey()))
                    colStats.putIfAbsent(oldColStat.getKey(), oldColStat.getValue());
            }
        }
        res = new ObjectPartitionStatisticsImpl(partId, getRowCount(colStats), locPart.updateCounter(), colStats);
    } finally {
        if (reserved)
            locPart.release();
    }
    statRepo.replaceLocalPartitionStatistics(gathCtx.configuration().key(), res);
    if (gathCtx.configuration().columns().size() == colsToCollect.size())
        statRepo.refreshObsolescence(gathCtx.configuration().key(), partId);
    return res;
}