Examples with EqualIntervalBinning ml.shifu.shifu.core.binning.EqualIntervalBinning used on opensource projects

Example 1 with EqualIntervalBinning

use of ml.shifu.shifu.core.binning.EqualIntervalBinning in project shifu by ShifuML.

the class BinningDataUDF method exec.

/*
     * (non-Javadoc)
     * 
     * @see org.apache.pig.EvalFunc#exec(org.apache.pig.data.Tuple)
     */
@Override
public Tuple exec(Tuple input) throws IOException {
    if (input == null || input.size() < 2) {
        return null;
    }
    Integer columnId = (Integer) input.get(0);
    DataBag databag = (DataBag) input.get(1);
    ColumnConfig columnConfig = super.columnConfigList.get(columnId);
    AbstractBinning<?> binning = null;
    if (columnConfig.isCategorical()) {
        binning = new CategoricalBinning(-1, super.modelConfig.getMissingOrInvalidValues(), this.maxCategorySize);
    } else {
        if (super.modelConfig.getBinningMethod().equals(BinningMethod.EqualInterval)) {
            binning = new EqualIntervalBinning(modelConfig.getStats().getMaxNumBin());
        } else {
            switch(this.modelConfig.getBinningAlgorithm()) {
                case Native:
                    log.info("Invoke Native binning method, memory cosuming!!");
                    // always merge bins
                    binning = new NativeBinning(modelConfig.getStats().getMaxNumBin(), true);
                    break;
                case SPDT:
                case SPDTI:
                    log.info("Invoke SPDT(Streaming Parallel Decision Tree) binning method, ");
                    binning = new EqualPopulationBinning(modelConfig.getStats().getMaxNumBin());
                    break;
                case MunroPat:
                case MunroPatI:
                    log.info("Invoke Munro & Paterson selecting algorithm");
                    binning = new MunroPatBinning(modelConfig.getStats().getMaxNumBin());
                    break;
                default:
                    log.info("Default: Invoke Munro & Paterson selecting algorithm");
                    binning = new MunroPatBinning(modelConfig.getStats().getMaxNumBin());
                    break;
            }
        }
    }
    Iterator<Tuple> iterator = databag.iterator();
    while (iterator.hasNext()) {
        Tuple element = iterator.next();
        if (element == null || element.size() < 2) {
            continue;
        }
        Object value = element.get(1);
        if (value != null) {
            binning.addData(value.toString());
        }
    }
    Tuple output = TupleFactory.getInstance().newTuple(2);
    output.set(0, columnId);
    // Do check here. It's because if there are too many value for categorical variable,
    // it will consume too much memory when join them together, that will cause OOM exception
    List<?> dataBin = binning.getDataBin();
    if (dataBin.size() > this.maxCategorySize) {
        output.set(1, "");
    } else {
        output.set(1, StringUtils.join(dataBin, CalculateStatsUDF.CATEGORY_VAL_SEPARATOR));
    }
    log.info("Finish merging bin info for columnId - " + columnId);
    return output;
}

Example 2 with EqualIntervalBinning

use of ml.shifu.shifu.core.binning.EqualIntervalBinning in project shifu by ShifuML.

the class DynamicBinningUDFTest method createDataBag.

private DataBag createDataBag() throws IOException {
    InputStream is = DynamicBinningUDFTest.class.getResourceAsStream("/example/inner_seg1_v15/dib_sample_fields.gz");
    GZIPInputStream gzis = new GZIPInputStream(is);
    DataBag databag = new DefaultDataBag();
    EqualIntervalBinning inst = new EqualIntervalBinning(1000);
    List<String> lines = IOUtils.readLines(gzis);
    for (String record : lines) {
        String[] fields = record.split("\\|");
        inst.addData(fields[0]);
        Tuple tuple = TupleFactory.getInstance().newTuple(4);
        tuple.set(0, 1);
        tuple.set(1, fields[0]);
        tuple.set(2, (fields[1].equals("1") ? Boolean.TRUE : Boolean.FALSE));
        tuple.set(3, 0);
        databag.add(tuple);
    }
    IOUtils.closeQuietly(gzis);
    IOUtils.closeQuietly(is);
    return databag;
}

Example 3 with EqualIntervalBinning

use of ml.shifu.shifu.core.binning.EqualIntervalBinning in project shifu by ShifuML.

the class BinningPartialDataUDF method exec.

/*
     * (non-Javadoc)
     * 
     * @see org.apache.pig.EvalFunc#exec(org.apache.pig.data.Tuple)
     */
@Override
public String exec(Tuple input) throws IOException {
    if (input == null) {
        return null;
    }
    DataBag databag = (DataBag) input.get(0);
    Iterator<Tuple> iterator = databag.iterator();
    while (iterator.hasNext()) {
        Tuple element = iterator.next();
        if (element == null) {
            continue;
        }
        if (columnId < 0) {
            columnId = (Integer) element.get(0);
            if (columnId >= super.columnConfigList.size()) {
                columnId = columnId % super.columnConfigList.size();
            }
            columnConfig = super.columnConfigList.get(columnId);
            if (columnConfig.isCategorical()) {
                binning = new CategoricalBinning(-1, modelConfig.getMissingOrInvalidValues(), this.maxCategorySize);
            } else {
                if (super.modelConfig.getBinningMethod().equals(BinningMethod.EqualInterval)) {
                    binning = new EqualIntervalBinning(modelConfig.getStats().getMaxNumBin() > 0 ? modelConfig.getStats().getMaxNumBin() : 1024, modelConfig.getMissingOrInvalidValues());
                } else {
                    binning = new EqualPopulationBinning(modelConfig.getStats().getMaxNumBin() > 0 ? modelConfig.getStats().getMaxNumBin() : 1024, modelConfig.getMissingOrInvalidValues());
                }
            }
            if (columnConfig.isHybrid()) {
                this.backUpbinning = new CategoricalBinning(-1, modelConfig.getMissingOrInvalidValues(), this.maxCategorySize);
            }
        }
        Object value = element.get(1);
        if (value != null) {
            String valStr = value.toString();
            if (isWeightBinningMethod() && binning instanceof EqualPopulationBinning) {
                ((EqualPopulationBinning) binning).addData(valStr, (Double) element.get(AddColumnNumUDF.COLUMN_WEIGHT_INDX));
            } else {
                binning.addData(valStr);
            }
            if (this.columnConfig.isHybrid()) {
                // missing value and not number value go to categorical binning
                double douVal = BinUtils.parseNumber(valStr);
                Double hybridThreshould = this.columnConfig.getHybridThreshold();
                if (hybridThreshould == null) {
                    hybridThreshould = Double.NEGATIVE_INFINITY;
                }
                // douVal < hybridThreshould which will also be set to category
                boolean isCategory = Double.isNaN(douVal) || douVal < hybridThreshould;
                if (douVal < hybridThreshould) {
                    log.warn("douVal " + douVal + ", threshold " + hybridThreshould + ", column {}" + columnConfig.getColumnName());
                }
                if (binning.isMissingVal(valStr) || isCategory) {
                    this.backUpbinning.addData(valStr);
                }
            }
        }
    }
    String binningObjStr = ((binning == null) ? null : binning.objToString());
    if (this.columnConfig.isHybrid()) {
        binningObjStr += Constants.HYBRID_BIN_STR_DILIMETER + this.backUpbinning.objToString();
    }
    cleanUp();
    return binningObjStr;
}