Examples with Statistics3Table org.knime.base.data.statistics.Statistics3Table used on opensource projects

Example 1 with Statistics3Table

use of org.knime.base.data.statistics.Statistics3Table in project knime-core by knime.

the class Normalizer2 method doDecimalScaling.

/**
 * Does the decimal scaling.
 *
 * @param exec an object to check for user cancellations. Can be <code>null</code>.
 * @throws CanceledExecutionException if user canceled
 * @return the normalized DataTable
 */
public AffineTransTable doDecimalScaling(final ExecutionContext exec) throws CanceledExecutionException {
    Statistics3Table st = new Statistics3Table(m_table, false, 0, Collections.<String>emptyList(), exec);
    checkForMissVals(st);
    String[] includes = getNames();
    double[] max = st.getMax();
    double[] min = st.getMin();
    double[] scales = new double[m_colindices.length];
    double[] transforms = new double[m_colindices.length];
    double[] mins = new double[m_colindices.length];
    double[] maxs = new double[m_colindices.length];
    for (int i = 0; i < m_colindices.length; i++) {
        int trueIndex = m_colindices[i];
        double absMax = Math.abs(max[trueIndex]);
        double absMin = Math.abs(min[trueIndex]);
        double maxvalue = absMax > absMin ? absMax : absMin;
        int exp = 0;
        // Unreported bug fix: when there was an infinite value, it takes infinite time to reach 1 by / 10.
        if (Double.isInfinite(maxvalue)) {
            throw new IllegalStateException("Cannot handle infinite values: " + includes[i]);
        }
        while (Math.abs(maxvalue) > 1) {
            maxvalue = maxvalue / DECIMAL_BASE;
            exp++;
        }
        scales[i] = 1.0 / Math.pow(DECIMAL_BASE, exp);
        transforms[i] = 0.0;
        mins[i] = -1.0;
        maxs[i] = 1.0;
    }
    String summary = "Decimal Scaling normalization on " + includes.length + " column(s)";
    AffineTransConfiguration configuration = new AffineTransConfiguration(includes, scales, transforms, mins, maxs, summary);
    return new AffineTransTable(m_table, configuration);
}

Example 2 with Statistics3Table

use of org.knime.base.data.statistics.Statistics3Table in project knime-core by knime.

the class Normalizer2 method doZScoreNorm.

/**
 * Does the Z-Score Normalization.
 *
 * @param exec an object to check for user cancelations. Can be <code>null</code>.
 * @throws CanceledExecutionException if user canceled
 * @return the normalized DataTable
 */
public AffineTransTable doZScoreNorm(final ExecutionContext exec) throws CanceledExecutionException {
    ExecutionContext statisticsExec = exec.createSubExecutionContext(.5);
    final Statistics3Table st = new Statistics3Table(m_table, false, 0, Collections.<String>emptyList(), statisticsExec);
    checkForMissVals(st);
    double[] mean = st.getMean();
    double[] stddev = st.getStandardDeviation();
    final double[] scales = new double[m_colindices.length];
    final double[] transforms = new double[m_colindices.length];
    final double[] mins = new double[m_colindices.length];
    final double[] maxs = new double[m_colindices.length];
    for (int i = 0; i < m_colindices.length; i++) {
        if (Double.isNaN(mean[m_colindices[i]])) {
            scales[i] = Double.NaN;
            transforms[i] = Double.NaN;
        } else {
            scales[i] = (stddev[m_colindices[i]] == 0.0 ? 1.0 : 1.0 / stddev[m_colindices[i]]);
            transforms[i] = -mean[m_colindices[i]] * scales[i];
        }
        mins[i] = Double.NaN;
        maxs[i] = Double.NaN;
    }
    String[] includes = getNames();
    String summary = "Z-Score (Gaussian) normalization on " + includes.length + " column(s)";
    AffineTransConfiguration configuration = new AffineTransConfiguration(includes, scales, transforms, mins, maxs, summary);
    return new AffineTransTable(m_table, configuration);
}

Example 3 with Statistics3Table

use of org.knime.base.data.statistics.Statistics3Table in project knime-core by knime.

the class Normalizer2 method doMinMaxNorm.

/**
 * Does the Min-Max Normalization.
 *
 * @param newmax the new maximum
 * @param newmin the new minimum
 * @param exec an object to check for user cancelations. Can be <code>null</code>.
 * @throws CanceledExecutionException if user canceled
 * @return normalized DataTable
 */
public AffineTransTable doMinMaxNorm(final double newmax, final double newmin, final ExecutionContext exec) throws CanceledExecutionException {
    ExecutionContext statisticsExec = exec.createSilentSubExecutionContext(.5);
    Statistics3Table st;
    st = new Statistics3Table(m_table, false, 0, Collections.<String>emptyList(), statisticsExec);
    checkForMissVals(st);
    DataTableSpec spec = m_table.getDataTableSpec();
    double[] max = st.getMax();
    double[] min = st.getMin();
    final double[] scales = new double[m_colindices.length];
    final double[] transforms = new double[m_colindices.length];
    final double[] mins = new double[m_colindices.length];
    final double[] maxs = new double[m_colindices.length];
    for (int i = 0; i < transforms.length; i++) {
        DataColumnSpec cSpec = spec.getColumnSpec(m_colindices[i]);
        boolean isDouble = cSpec.getType().isCompatible(DoubleValue.class);
        if (!isDouble) {
            assert (!isDouble);
            scales[i] = Double.NaN;
            transforms[i] = Double.NaN;
            mins[i] = Double.NaN;
            maxs[i] = Double.NaN;
        } else {
            // scales and translation to [0,1]
            double maxI = max[m_colindices[i]];
            double minI = min[m_colindices[i]];
            scales[i] = (maxI == minI ? 1 : 1.0 / (maxI - minI));
            transforms[i] = -minI * scales[i];
            // scale and translation to [newmin, newmax]
            scales[i] *= (newmax - newmin);
            transforms[i] *= (newmax - newmin);
            transforms[i] += newmin;
            mins[i] = newmin;
            maxs[i] = newmax;
        }
    }
    String[] includes = getNames();
    String minS = DoubleFormat.formatDouble(newmin);
    String maxS = DoubleFormat.formatDouble(newmax);
    String summary = "Min/Max (" + minS + ", " + maxS + ") normalization " + "on " + includes.length + " column(s)";
    AffineTransConfiguration configuration = new AffineTransConfiguration(includes, scales, transforms, mins, maxs, summary);
    return new AffineTransTable(m_table, configuration);
}

Example 4 with Statistics3Table

use of org.knime.base.data.statistics.Statistics3Table in project knime-core by knime.

the class Statistics3NodeModel method execute.

/**
 * Computes the statistics for the DataTable at the in-port. Use the view on
 * this node to see them.
 *
 * {@inheritDoc}
 */
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws CanceledExecutionException {
    m_statTable = new Statistics3Table(inData[0], m_computeMedian.getBooleanValue(), numOfNominalValuesOutput(), m_nominalFilter.getIncludeList(), exec);
    if (getStatTable().getWarning() != null) {
        super.setWarningMessage(getStatTable().getWarning());
    }
    BufferedDataTable outTable1 = exec.createBufferedDataTable(getStatTable().createStatisticMomentsTable(), exec.createSubProgress(0.5));
    BufferedDataTable outTable2 = exec.createBufferedDataTable(getStatTable().createNominalValueTable(m_nominalFilter.getIncludeList()), exec.createSubProgress(0.5));
    return new BufferedDataTable[] { outTable1, outTable2 };
}

Example 5 with Statistics3Table

use of org.knime.base.data.statistics.Statistics3Table in project knime-core by knime.

the class ColorManager2NodeModel method execute.

/**
 * Is invoked during the node's execution to make the color settings.
 *
 * @param data the input data array
 * @param exec the execution monitor
 * @return the same input data table whereby the RowKeys contain color info
 *         now
 * @throws CanceledExecutionException if user canceled execution
 */
@Override
protected PortObject[] execute(final PortObject[] data, final ExecutionContext exec) throws CanceledExecutionException {
    assert (data != null && data.length == 1 && data[INPORT] != null);
    BufferedDataTable in = (BufferedDataTable) data[0];
    DataTableSpec inSpec = in.getDataTableSpec();
    ColorHandler colorHandler;
    // if no column has been selected, guess first nominal column
    if (m_column == null) {
        // find first nominal column with possible values
        String column = DataTableSpec.guessNominalClassColumn(inSpec, false);
        m_columnGuess = column;
        m_isNominalGuess = true;
        super.setWarningMessage("Selected column \"" + column + "\" with default nominal color mapping.");
        Set<DataCell> set = inSpec.getColumnSpec(column).getDomain().getValues();
        m_mapGuess.clear();
        m_mapGuess.putAll(ColorManager2DialogNominal.createColorMapping(set));
        colorHandler = createNominalColorHandler(m_mapGuess);
        DataTableSpec newSpec = getOutSpec(inSpec, column, colorHandler);
        BufferedDataTable changedSpecTable = exec.createSpecReplacerTable(in, newSpec);
        DataTableSpec modelSpec = new DataTableSpec(newSpec.getColumnSpec(column));
        ColorHandlerPortObject viewModel = new ColorHandlerPortObject(modelSpec, colorHandler.toString() + " based on column \"" + m_column + "\"");
        return new PortObject[] { changedSpecTable, viewModel };
    }
    // find column index
    int columnIndex = inSpec.findColumnIndex(m_column);
    // create new column spec based on color settings
    DataColumnSpec cspec = inSpec.getColumnSpec(m_column);
    if (m_isNominal) {
        colorHandler = createNominalColorHandler(m_map);
    } else {
        DataColumnDomain dom = cspec.getDomain();
        DataCell lower, upper;
        if (dom.hasBounds()) {
            lower = dom.getLowerBound();
            upper = dom.getUpperBound();
        } else {
            Statistics3Table stat = new Statistics3Table(in, false, 0, Collections.<String>emptyList(), exec);
            lower = stat.getMinCells()[columnIndex];
            upper = stat.getMaxCells()[columnIndex];
        }
        colorHandler = createRangeColorHandler(lower, upper, m_map);
    }
    DataTableSpec newSpec = getOutSpec(inSpec, m_column, colorHandler);
    DataTableSpec modelSpec = new DataTableSpec(newSpec.getColumnSpec(m_column));
    BufferedDataTable changedSpecTable = exec.createSpecReplacerTable(in, newSpec);
    ColorHandlerPortObject viewModel = new ColorHandlerPortObject(modelSpec, "Coloring on \"" + m_column + "\"");
    return new PortObject[] { changedSpecTable, viewModel };
}