Examples with CardinalityMergeException com.clearspring.analytics.stream.cardinality.CardinalityMergeException used on opensource projects

Example 1 with CardinalityMergeException

use of com.clearspring.analytics.stream.cardinality.CardinalityMergeException in project pinot by linkedin.

the class FastHLLAggregationFunction method aggregate.

@Override
public void aggregate(int length, @Nonnull AggregationResultHolder aggregationResultHolder, @Nonnull BlockValSet... blockValSets) {
    String[] valueArray = blockValSets[0].getStringValuesSV();
    HyperLogLog hyperLogLog = aggregationResultHolder.getResult();
    if (hyperLogLog == null) {
        hyperLogLog = new HyperLogLog(_log2m);
        aggregationResultHolder.setValue(hyperLogLog);
    }
    for (int i = 0; i < length; i++) {
        try {
            hyperLogLog.addAll(HllUtil.convertStringToHll(valueArray[i]));
        } catch (CardinalityMergeException e) {
            throw new RuntimeException("Caught exception while aggregating HyperLogLog.", e);
        }
    }
}

Example 2 with CardinalityMergeException

use of com.clearspring.analytics.stream.cardinality.CardinalityMergeException in project pinot by linkedin.

the class FastHLLAggregationFunction method aggregateGroupByMV.

@Override
public void aggregateGroupByMV(int length, @Nonnull int[][] groupKeysArray, @Nonnull GroupByResultHolder groupByResultHolder, @Nonnull BlockValSet... blockValSets) {
    String[] valueArray = blockValSets[0].getStringValuesSV();
    for (int i = 0; i < length; i++) {
        String value = valueArray[i];
        for (int groupKey : groupKeysArray[i]) {
            HyperLogLog hyperLogLog = groupByResultHolder.getResult(groupKey);
            if (hyperLogLog == null) {
                hyperLogLog = new HyperLogLog(_log2m);
                groupByResultHolder.setValueForKey(groupKey, hyperLogLog);
            }
            try {
                hyperLogLog.addAll(HllUtil.convertStringToHll(value));
            } catch (CardinalityMergeException e) {
                throw new RuntimeException("Caught exception while aggregating HyperLogLog.", e);
            }
        }
    }
}

Example 3 with CardinalityMergeException

use of com.clearspring.analytics.stream.cardinality.CardinalityMergeException in project pinot by linkedin.

the class FastHLLMVAggregationFunction method aggregateGroupByMV.

@Override
public void aggregateGroupByMV(int length, @Nonnull int[][] groupKeysArray, @Nonnull GroupByResultHolder groupByResultHolder, @Nonnull BlockValSet... blockValSets) {
    String[][] valuesArray = blockValSets[0].getStringValuesMV();
    for (int i = 0; i < length; i++) {
        String[] values = valuesArray[i];
        for (int groupKey : groupKeysArray[i]) {
            HyperLogLog hyperLogLog = groupByResultHolder.getResult(groupKey);
            if (hyperLogLog == null) {
                hyperLogLog = new HyperLogLog(_log2m);
                groupByResultHolder.setValueForKey(groupKey, hyperLogLog);
            }
            try {
                for (String value : values) {
                    hyperLogLog.addAll(HllUtil.convertStringToHll(value));
                }
            } catch (CardinalityMergeException e) {
                throw new RuntimeException("Caught exception while aggregating HyperLogLog.", e);
            }
        }
    }
}

Example 4 with CardinalityMergeException

use of com.clearspring.analytics.stream.cardinality.CardinalityMergeException in project pinot by linkedin.

the class HllUtil method clone.

public static HyperLogLog clone(HyperLogLog hll, int log2m) {
    try {
        HyperLogLog ret = new HyperLogLog(log2m);
        ret.addAll(hll);
        return ret;
    } catch (CardinalityMergeException e) {
        throw new RuntimeException(e);
    }
}

Example 5 with CardinalityMergeException

use of com.clearspring.analytics.stream.cardinality.CardinalityMergeException in project shifu by ShifuML.

the class UpdateBinningInfoReducer method reduce.

@Override
protected void reduce(IntWritable key, Iterable<BinningInfoWritable> values, Context context) throws IOException, InterruptedException {
    long start = System.currentTimeMillis();
    double sum = 0d;
    double squaredSum = 0d;
    double tripleSum = 0d;
    double quarticSum = 0d;
    double p25th = 0d;
    double median = 0d;
    double p75th = 0d;
    long count = 0L, missingCount = 0L;
    double min = Double.MAX_VALUE, max = Double.MIN_VALUE;
    List<Double> binBoundaryList = null;
    List<String> binCategories = null;
    long[] binCountPos = null;
    long[] binCountNeg = null;
    double[] binWeightPos = null;
    double[] binWeightNeg = null;
    long[] binCountTotal = null;
    int columnConfigIndex = key.get() >= this.columnConfigList.size() ? key.get() % this.columnConfigList.size() : key.get();
    ColumnConfig columnConfig = this.columnConfigList.get(columnConfigIndex);
    HyperLogLogPlus hyperLogLogPlus = null;
    Set<String> fis = new HashSet<String>();
    long totalCount = 0, invalidCount = 0, validNumCount = 0;
    int binSize = 0;
    for (BinningInfoWritable info : values) {
        if (info.isEmpty()) {
            // mapper has no stats, skip it
            continue;
        }
        CountAndFrequentItemsWritable cfiw = info.getCfiw();
        totalCount += cfiw.getCount();
        invalidCount += cfiw.getInvalidCount();
        validNumCount += cfiw.getValidNumCount();
        fis.addAll(cfiw.getFrequetItems());
        if (hyperLogLogPlus == null) {
            hyperLogLogPlus = HyperLogLogPlus.Builder.build(cfiw.getHyperBytes());
        } else {
            try {
                hyperLogLogPlus = (HyperLogLogPlus) hyperLogLogPlus.merge(HyperLogLogPlus.Builder.build(cfiw.getHyperBytes()));
            } catch (CardinalityMergeException e) {
                throw new RuntimeException(e);
            }
        }
        if (columnConfig.isHybrid() && binBoundaryList == null && binCategories == null) {
            binBoundaryList = info.getBinBoundaries();
            binCategories = info.getBinCategories();
            binSize = binBoundaryList.size() + binCategories.size();
            binCountPos = new long[binSize + 1];
            binCountNeg = new long[binSize + 1];
            binWeightPos = new double[binSize + 1];
            binWeightNeg = new double[binSize + 1];
            binCountTotal = new long[binSize + 1];
        } else if (columnConfig.isNumerical() && binBoundaryList == null) {
            binBoundaryList = info.getBinBoundaries();
            binSize = binBoundaryList.size();
            binCountPos = new long[binSize + 1];
            binCountNeg = new long[binSize + 1];
            binWeightPos = new double[binSize + 1];
            binWeightNeg = new double[binSize + 1];
            binCountTotal = new long[binSize + 1];
        } else if (columnConfig.isCategorical() && binCategories == null) {
            binCategories = info.getBinCategories();
            binSize = binCategories.size();
            binCountPos = new long[binSize + 1];
            binCountNeg = new long[binSize + 1];
            binWeightPos = new double[binSize + 1];
            binWeightNeg = new double[binSize + 1];
            binCountTotal = new long[binSize + 1];
        }
        count += info.getTotalCount();
        missingCount += info.getMissingCount();
        // for numeric, such sums are OK, for categorical, such values are all 0, should be updated by using
        // binCountPos and binCountNeg
        sum += info.getSum();
        squaredSum += info.getSquaredSum();
        tripleSum += info.getTripleSum();
        quarticSum += info.getQuarticSum();
        if (Double.compare(max, info.getMax()) < 0) {
            max = info.getMax();
        }
        if (Double.compare(min, info.getMin()) > 0) {
            min = info.getMin();
        }
        for (int i = 0; i < (binSize + 1); i++) {
            binCountPos[i] += info.getBinCountPos()[i];
            binCountNeg[i] += info.getBinCountNeg()[i];
            binWeightPos[i] += info.getBinWeightPos()[i];
            binWeightNeg[i] += info.getBinWeightNeg()[i];
            binCountTotal[i] += info.getBinCountPos()[i];
            binCountTotal[i] += info.getBinCountNeg()[i];
        }
    }
    if (columnConfig.isNumerical()) {
        long p25Count = count / 4;
        long medianCount = p25Count * 2;
        long p75Count = p25Count * 3;
        p25th = min;
        median = min;
        p75th = min;
        int currentCount = 0;
        for (int i = 0; i < binBoundaryList.size(); i++) {
            double left = getCutoffBoundary(binBoundaryList.get(i), max, min);
            double right = ((i == binBoundaryList.size() - 1) ? max : getCutoffBoundary(binBoundaryList.get(i + 1), max, min));
            if (p25Count >= currentCount && p25Count < currentCount + binCountTotal[i]) {
                p25th = ((p25Count - currentCount) / (double) binCountTotal[i]) * (right - left) + left;
            }
            if (medianCount >= currentCount && medianCount < currentCount + binCountTotal[i]) {
                median = ((medianCount - currentCount) / (double) binCountTotal[i]) * (right - left) + left;
            }
            if (p75Count >= currentCount && p75Count < currentCount + binCountTotal[i]) {
                p75th = ((p75Count - currentCount) / (double) binCountTotal[i]) * (right - left) + left;
                // when get 75 percentile stop it
                break;
            }
            currentCount += binCountTotal[i];
        }
        LOG.info("Coloumn num is {}, p25 value is {}, median value is {}, p75 value is {}", columnConfig.getColumnNum(), p25th, median, p75th);
    }
    LOG.info("Coloumn num is {}, columnType value is {}, cateMaxNumBin is {}, binCategory size is {}", columnConfig.getColumnNum(), columnConfig.getColumnType(), modelConfig.getStats().getCateMaxNumBin(), (CollectionUtils.isNotEmpty(columnConfig.getBinCategory()) ? columnConfig.getBinCategory().size() : 0));
    // To merge categorical binning
    if (columnConfig.isCategorical() && modelConfig.getStats().getCateMaxNumBin() > 0 && CollectionUtils.isNotEmpty(binCategories) && binCategories.size() > modelConfig.getStats().getCateMaxNumBin()) {
        // only category size large then expected max bin number
        CateBinningStats cateBinningStats = rebinCategoricalValues(new CateBinningStats(binCategories, binCountPos, binCountNeg, binWeightPos, binWeightNeg));
        LOG.info("For variable - {}, {} bins is rebined to {} bins", columnConfig.getColumnName(), binCategories.size(), cateBinningStats.binCategories.size());
        binCategories = cateBinningStats.binCategories;
        binCountPos = cateBinningStats.binCountPos;
        binCountNeg = cateBinningStats.binCountNeg;
        binWeightPos = cateBinningStats.binWeightPos;
        binWeightNeg = cateBinningStats.binWeightNeg;
    }
    double[] binPosRate;
    if (modelConfig.isRegression()) {
        binPosRate = computePosRate(binCountPos, binCountNeg);
    } else {
        // for multiple classfication, use rate of categories to compute a value
        binPosRate = computeRateForMultiClassfication(binCountPos);
    }
    String binBounString = null;
    if (columnConfig.isHybrid()) {
        if (binCategories.size() > this.maxCateSize) {
            LOG.warn("Column {} {} with invalid bin category size.", key.get(), columnConfig.getColumnName(), binCategories.size());
            return;
        }
        binBounString = binBoundaryList.toString();
        binBounString += Constants.HYBRID_BIN_STR_DILIMETER + Base64Utils.base64Encode("[" + StringUtils.join(binCategories, CalculateStatsUDF.CATEGORY_VAL_SEPARATOR) + "]");
    } else if (columnConfig.isCategorical()) {
        if (binCategories.size() > this.maxCateSize) {
            LOG.warn("Column {} {} with invalid bin category size.", key.get(), columnConfig.getColumnName(), binCategories.size());
            return;
        }
        binBounString = Base64Utils.base64Encode("[" + StringUtils.join(binCategories, CalculateStatsUDF.CATEGORY_VAL_SEPARATOR) + "]");
        // recompute such value for categorical variables
        min = Double.MAX_VALUE;
        max = Double.MIN_VALUE;
        sum = 0d;
        squaredSum = 0d;
        for (int i = 0; i < binPosRate.length; i++) {
            if (!Double.isNaN(binPosRate[i])) {
                if (Double.compare(max, binPosRate[i]) < 0) {
                    max = binPosRate[i];
                }
                if (Double.compare(min, binPosRate[i]) > 0) {
                    min = binPosRate[i];
                }
                long binCount = binCountPos[i] + binCountNeg[i];
                sum += binPosRate[i] * binCount;
                double squaredVal = binPosRate[i] * binPosRate[i];
                squaredSum += squaredVal * binCount;
                tripleSum += squaredVal * binPosRate[i] * binCount;
                quarticSum += squaredVal * squaredVal * binCount;
            }
        }
    } else {
        if (binBoundaryList.size() == 0) {
            LOG.warn("Column {} {} with invalid bin boundary size.", key.get(), columnConfig.getColumnName(), binBoundaryList.size());
            return;
        }
        binBounString = binBoundaryList.toString();
    }
    ColumnMetrics columnCountMetrics = null;
    ColumnMetrics columnWeightMetrics = null;
    if (modelConfig.isRegression()) {
        columnCountMetrics = ColumnStatsCalculator.calculateColumnMetrics(binCountNeg, binCountPos);
        columnWeightMetrics = ColumnStatsCalculator.calculateColumnMetrics(binWeightNeg, binWeightPos);
    }
    // To make it be consistent with SPDT, missingCount is excluded to compute mean, stddev ...
    long realCount = this.statsExcludeMissingValue ? (count - missingCount) : count;
    double mean = sum / realCount;
    double stdDev = Math.sqrt(Math.abs((squaredSum - (sum * sum) / realCount + EPS) / (realCount - 1)));
    double aStdDev = Math.sqrt(Math.abs((squaredSum - (sum * sum) / realCount + EPS) / realCount));
    double skewness = ColumnStatsCalculator.computeSkewness(realCount, mean, aStdDev, sum, squaredSum, tripleSum);
    double kurtosis = ColumnStatsCalculator.computeKurtosis(realCount, mean, aStdDev, sum, squaredSum, tripleSum, quarticSum);
    sb.append(key.get()).append(Constants.DEFAULT_DELIMITER).append(binBounString).append(Constants.DEFAULT_DELIMITER).append(Arrays.toString(binCountNeg)).append(Constants.DEFAULT_DELIMITER).append(Arrays.toString(binCountPos)).append(Constants.DEFAULT_DELIMITER).append(Arrays.toString(new double[0])).append(Constants.DEFAULT_DELIMITER).append(Arrays.toString(binPosRate)).append(Constants.DEFAULT_DELIMITER).append(columnCountMetrics == null ? "" : df.format(columnCountMetrics.getKs())).append(Constants.DEFAULT_DELIMITER).append(columnCountMetrics == null ? "" : df.format(columnCountMetrics.getIv())).append(Constants.DEFAULT_DELIMITER).append(df.format(max)).append(Constants.DEFAULT_DELIMITER).append(df.format(min)).append(Constants.DEFAULT_DELIMITER).append(df.format(mean)).append(Constants.DEFAULT_DELIMITER).append(df.format(stdDev)).append(Constants.DEFAULT_DELIMITER).append(columnConfig.getColumnType().toString()).append(Constants.DEFAULT_DELIMITER).append(median).append(Constants.DEFAULT_DELIMITER).append(missingCount).append(Constants.DEFAULT_DELIMITER).append(count).append(Constants.DEFAULT_DELIMITER).append(missingCount * 1.0d / count).append(Constants.DEFAULT_DELIMITER).append(Arrays.toString(binWeightNeg)).append(Constants.DEFAULT_DELIMITER).append(Arrays.toString(binWeightPos)).append(Constants.DEFAULT_DELIMITER).append(columnCountMetrics == null ? "" : columnCountMetrics.getWoe()).append(Constants.DEFAULT_DELIMITER).append(columnWeightMetrics == null ? "" : columnWeightMetrics.getWoe()).append(Constants.DEFAULT_DELIMITER).append(columnWeightMetrics == null ? "" : columnWeightMetrics.getKs()).append(Constants.DEFAULT_DELIMITER).append(columnWeightMetrics == null ? "" : columnWeightMetrics.getIv()).append(Constants.DEFAULT_DELIMITER).append(columnCountMetrics == null ? Arrays.toString(new double[binSize + 1]) : columnCountMetrics.getBinningWoe().toString()).append(Constants.DEFAULT_DELIMITER).append(columnWeightMetrics == null ? Arrays.toString(new double[binSize + 1]) : // bin weighted WOE
    columnWeightMetrics.getBinningWoe().toString()).append(Constants.DEFAULT_DELIMITER).append(// skewness
    skewness).append(Constants.DEFAULT_DELIMITER).append(// kurtosis
    kurtosis).append(Constants.DEFAULT_DELIMITER).append(// total count
    totalCount).append(Constants.DEFAULT_DELIMITER).append(// invalid count
    invalidCount).append(Constants.DEFAULT_DELIMITER).append(// valid num count
    validNumCount).append(Constants.DEFAULT_DELIMITER).append(// cardinality
    hyperLogLogPlus.cardinality()).append(Constants.DEFAULT_DELIMITER).append(// frequent items
    Base64Utils.base64Encode(limitedFrequentItems(fis))).append(Constants.DEFAULT_DELIMITER).append(// the 25 percentile value
    p25th).append(Constants.DEFAULT_DELIMITER).append(p75th);
    outputValue.set(sb.toString());
    context.write(NullWritable.get(), outputValue);
    sb.delete(0, sb.length());
    LOG.debug("Time:{}", (System.currentTimeMillis() - start));
}