Examples with DoubleValue org.knime.core.data.DoubleValue used on opensource projects

Example 91 with DoubleValue

use of org.knime.core.data.DoubleValue in project knime-core by knime.

the class FilterRowIteratorTest method check.

/*
     * Used for all test* methods to check if all only these c's appear in the
     * filer. @param d Initial array of possible double values. @param c An
     * array of double values to check.
     */
private void check(final double[] d, final double[] c) {
    FilterRowGenerator gen = new MyFilterRowGenerator(c);
    RowIterator it = new FilterRowIterator(new MyRowIterator(d), gen);
    ArrayList<Double> list = new ArrayList<Double>();
    // copy c's in list to remove them easily
    for (int i = 0; i < c.length; i++) {
        list.add(c[i]);
    }
    // check all r's in list of c's
    for (int i = 0; it.hasNext(); i++) {
        DataRow row = it.next();
        double r = ((DoubleValue) row.getCell(0)).getDoubleValue();
        assertTrue(list.remove(r));
    }
    // check for no d's in list
    for (int i = 0; i < d.length; i++) {
        assertFalse(list.remove(d[i]));
    }
}

Example 92 with DoubleValue

use of org.knime.core.data.DoubleValue in project knime-core by knime.

the class StatisticCalculatorTest method doubleMedianTest.

/**
 * Tests the median with double cells.
 *
 * @throws Exception e
 */
@Test
public void doubleMedianTest() throws Exception {
    // create some random tables with random missing values
    for (int i = 0; i < 50; i++) {
        final BufferedDataTable table = createRandomTableWithMissingValues(4, 100);
        Statistics3Table statistics3Table = new Statistics3Table(table, true, 0, Collections.<String>emptyList(), EXEC_CONTEXT, ascendingIntArray(4));
        Median median = new Median();
        StatisticCalculator statisticCalculator = new StatisticCalculator(table.getDataTableSpec(), table.getDataTableSpec().getColumnNames(), median);
        statisticCalculator.evaluate(table, EXEC_CONTEXT);
        for (int j = 0; j < 4; j++) {
            double oldMed = statistics3Table.getMedian(j);
            double newMed = ((DoubleValue) median.getMedian(table.getDataTableSpec().getColumnSpec(j).getName())).getDoubleValue();
            assertEquals(oldMed, newMed, 0.00001);
        }
    }
}

Example 93 with DoubleValue

use of org.knime.core.data.DoubleValue in project knime-core by knime.

the class EnrichmentPlotterModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws Exception {
    final double rowCount = inData[0].size();
    final BufferedDataContainer areaOutCont = exec.createDataContainer(AREA_OUT_SPEC);
    final BufferedDataContainer discrateOutCont = exec.createDataContainer(getDiscrateOutSpec());
    final double[] fractionSizes = m_settings.getFractionSizes();
    for (int i = 0; i < m_settings.getCurveCount(); i++) {
        final ExecutionMonitor sexec = exec.createSubProgress(1.0 / m_settings.getCurveCount());
        exec.setMessage("Generating curve " + (i + 1));
        final Curve c = m_settings.getCurve(i);
        final Helper[] curve = new Helper[KnowsRowCountTable.checkRowCount(inData[0].size())];
        final int sortIndex = inData[0].getDataTableSpec().findColumnIndex(c.getSortColumn());
        final int actIndex = inData[0].getDataTableSpec().findColumnIndex(c.getActivityColumn());
        int k = 0, maxK = 0;
        for (DataRow row : inData[0]) {
            DataCell c1 = row.getCell(sortIndex);
            DataCell c2 = row.getCell(actIndex);
            if (k++ % 100 == 0) {
                sexec.checkCanceled();
                sexec.setProgress(k / rowCount);
            }
            if (c1.isMissing()) {
                continue;
            } else {
                curve[maxK] = new Helper(((DoubleValue) c1).getDoubleValue(), c2);
            }
            maxK++;
        }
        Arrays.sort(curve, 0, maxK);
        if (c.isSortDescending()) {
            for (int j = 0; j < maxK / 2; j++) {
                Helper h = curve[j];
                curve[j] = curve[maxK - j - 1];
                curve[maxK - j - 1] = h;
            }
        }
        // this is for down-sampling so that the view is faster;
        // plotting >100,000 points takes quite a long time
        final int size = Math.min(MAX_RESOLUTION, maxK);
        final double downSampleRate = maxK / (double) size;
        final double[] xValues = new double[size + 1];
        final double[] yValues = new double[size + 1];
        xValues[0] = 0;
        yValues[0] = 0;
        int lastK = 0;
        double y = 0, area = 0;
        int nextHitRatePoint = 0;
        final double[] hitRateValues = new double[fractionSizes.length];
        final HashMap<DataCell, MutableInteger> clusters = new HashMap<DataCell, MutableInteger>();
        // set hit rate values for fractions that are smaller than 1 row to 0
        while ((maxK * fractionSizes[nextHitRatePoint] / 100) < 1) {
            hitRateValues[nextHitRatePoint++] = 0;
        }
        for (k = 1; k <= maxK; k++) {
            final Helper h = curve[k - 1];
            if (m_settings.plotMode() == PlotMode.PlotSum) {
                y += ((DoubleValue) h.b).getDoubleValue();
            } else if (m_settings.plotMode() == PlotMode.PlotHits) {
                if (!h.b.isMissing() && (((DoubleValue) h.b).getDoubleValue() >= m_settings.hitThreshold())) {
                    y++;
                }
            } else if (!h.b.isMissing()) {
                MutableInteger count = clusters.get(h.b);
                if (count == null) {
                    count = new MutableInteger(0);
                    clusters.put(h.b, count);
                }
                if (count.inc() == m_settings.minClusterMembers()) {
                    y++;
                }
            }
            area += y / maxK;
            if ((int) (k / downSampleRate) >= lastK + 1) {
                lastK++;
                xValues[lastK] = k;
                yValues[lastK] = y;
            }
            // thats why this needs to be a while
            while ((nextHitRatePoint < fractionSizes.length) && (k == (int) Math.floor(maxK * fractionSizes[nextHitRatePoint] / 100))) {
                hitRateValues[nextHitRatePoint] = y;
                nextHitRatePoint++;
            }
        }
        xValues[xValues.length - 1] = maxK;
        yValues[yValues.length - 1] = y;
        area /= y;
        m_curves.add(new EnrichmentPlot(c.getSortColumn() + " vs " + c.getActivityColumn(), xValues, yValues, area));
        areaOutCont.addRowToTable(new DefaultRow(new RowKey(c.toString()), new DoubleCell(area)));
        for (int j = 0; j < hitRateValues.length; j++) {
            hitRateValues[j] /= y;
        }
        double[] enrichmentFactors = new double[hitRateValues.length];
        for (int j = 0; j < enrichmentFactors.length; j++) {
            enrichmentFactors[j] = calculateEnrichmentFactor(hitRateValues[j], fractionSizes[j]);
        }
        discrateOutCont.addRowToTable(new DefaultRow(new RowKey(c.toString()), ArrayUtils.addAll(hitRateValues, enrichmentFactors)));
    }
    areaOutCont.close();
    discrateOutCont.close();
    return new BufferedDataTable[] { areaOutCont.getTable(), discrateOutCont.getTable() };
}

Example 94 with DoubleValue

use of org.knime.core.data.DoubleValue in project knime-core by knime.

the class BoxPlotNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws Exception {
    if (inData[0] == null) {
        return new BufferedDataTable[] {};
    }
    BufferedDataTable table = inData[0];
    m_statistics = new LinkedHashMap<DataColumnSpec, double[]>();
    m_mildOutliers = new LinkedHashMap<String, Map<Double, Set<RowKey>>>();
    m_extremeOutliers = new LinkedHashMap<String, Map<Double, Set<RowKey>>>();
    int colIdx = 0;
    List<DataColumnSpec> outputColSpecs = new ArrayList<DataColumnSpec>();
    double subProgress = 1.0 / getNumNumericColumns(table.getDataTableSpec());
    for (DataColumnSpec colSpec : table.getDataTableSpec()) {
        ExecutionContext colExec = exec.createSubExecutionContext(subProgress);
        exec.checkCanceled();
        if (colSpec.getType().isCompatible(DoubleValue.class)) {
            double[] statistic = new double[SIZE];
            outputColSpecs.add(colSpec);
            List<String> col = new ArrayList<String>();
            col.add(colSpec.getName());
            ExecutionContext sortExec = colExec.createSubExecutionContext(0.75);
            ExecutionContext findExec = colExec.createSubExecutionContext(0.25);
            SortedTable sorted = new SortedTable(table, col, new boolean[] { true }, sortExec);
            long currRowAbsolute = 0;
            int currCountingRow = 1;
            double lastValue = 1;
            long nrOfRows = table.size();
            boolean first = true;
            for (DataRow row : sorted) {
                exec.checkCanceled();
                double rowProgress = currRowAbsolute / (double) table.size();
                findExec.setProgress(rowProgress, "determining statistics for: " + table.getDataTableSpec().getColumnSpec(colIdx).getName());
                if (row.getCell(colIdx).isMissing()) {
                    // asserts that the missing values are sorted at
                    // the top of the table
                    currRowAbsolute++;
                    nrOfRows--;
                    continue;
                }
                // get the first value = actually observed minimum
                if (first) {
                    statistic[MIN] = ((DoubleValue) row.getCell(colIdx)).getDoubleValue();
                    // initialize the statistics with first value
                    // if the table is large enough it will be overriden
                    // this is just for the case of tables with < 5 rows
                    statistic[MEDIAN] = statistic[MIN];
                    statistic[LOWER_QUARTILE] = statistic[MIN];
                    statistic[UPPER_QUARTILE] = statistic[MIN];
                    first = false;
                }
                // get the last value = actually observed maximum
                if (currRowAbsolute == table.size() - 1) {
                    statistic[MAX] = ((DoubleValue) row.getCell(colIdx)).getDoubleValue();
                }
                float medianPos = nrOfRows * 0.5f;
                float lowerQuartilePos = nrOfRows * 0.25f;
                float upperQuartilePos = nrOfRows * 0.75f;
                if (currCountingRow == (int) Math.floor(lowerQuartilePos) + 1) {
                    if (lowerQuartilePos % 1 != 0) {
                        // get the row's value
                        statistic[LOWER_QUARTILE] = ((DoubleValue) row.getCell(colIdx)).getDoubleValue();
                    } else {
                        // calculate the mean between row and last row
                        double value = ((DoubleValue) row.getCell(colIdx)).getDoubleValue();
                        statistic[LOWER_QUARTILE] = (value + lastValue) / 2;
                    }
                }
                if (currCountingRow == (int) Math.floor(medianPos) + 1) {
                    if (medianPos % 1 != 0) {
                        // get the row's value
                        statistic[MEDIAN] = ((DoubleValue) row.getCell(colIdx)).getDoubleValue();
                    } else {
                        // calculate the mean between row and last row
                        double value = ((DoubleValue) row.getCell(colIdx)).getDoubleValue();
                        statistic[MEDIAN] = (value + lastValue) / 2;
                    }
                }
                if (currCountingRow == (int) Math.floor(upperQuartilePos) + 1) {
                    if (upperQuartilePos % 1 != 0) {
                        // get the row's value
                        statistic[UPPER_QUARTILE] = ((DoubleValue) row.getCell(colIdx)).getDoubleValue();
                    } else {
                        // calculate the mean between row and last row
                        double value = ((DoubleValue) row.getCell(colIdx)).getDoubleValue();
                        statistic[UPPER_QUARTILE] = (value + lastValue) / 2;
                    }
                }
                lastValue = ((DoubleValue) row.getCell(colIdx)).getDoubleValue();
                currRowAbsolute++;
                currCountingRow++;
            }
            double iqr = statistic[UPPER_QUARTILE] - statistic[LOWER_QUARTILE];
            Map<Double, Set<RowKey>> mild = new LinkedHashMap<Double, Set<RowKey>>();
            Map<Double, Set<RowKey>> extreme = new LinkedHashMap<Double, Set<RowKey>>();
            // per default the whiskers are at min and max
            double[] whiskers = new double[] { statistic[MIN], statistic[MAX] };
            if (statistic[MIN] < (statistic[LOWER_QUARTILE] - (1.5 * iqr)) || statistic[MAX] > statistic[UPPER_QUARTILE] + (1.5 * iqr)) {
                detectOutliers(sorted, iqr, new double[] { statistic[LOWER_QUARTILE], statistic[UPPER_QUARTILE] }, mild, extreme, whiskers, colIdx);
            }
            statistic[LOWER_WHISKER] = whiskers[0];
            statistic[UPPER_WHISKER] = whiskers[1];
            m_mildOutliers.put(colSpec.getName(), mild);
            m_extremeOutliers.put(colSpec.getName(), extreme);
            m_statistics.put(colSpec, statistic);
        }
        colIdx++;
    }
    DataContainer container = createOutputTable(exec, outputColSpecs);
    // return a data array with just one row but with the data table spec
    // for the column selection panel
    m_array = new DefaultDataArray(table, 1, 2);
    return new BufferedDataTable[] { exec.createBufferedDataTable(container.getTable(), exec) };
}

Example 95 with DoubleValue

use of org.knime.core.data.DoubleValue in project knime-core by knime.

the class BoxplotCalculator method calculateMultipleConditional.

/**
 * Calculates statistics for a conditional box plot.
 * @param table the data table
 * @param catCol the column with the category values
 * @param numCol the numeric column
 * @param exec an execution context
 * @return A linked hash map with BoxplotStatistics for each category
 * @throws CanceledExecutionException when the user cancels the execution
 * @throws InvalidSettingsException when the category column has no domain values
 */
public LinkedHashMap<String, LinkedHashMap<String, BoxplotStatistics>> calculateMultipleConditional(final BufferedDataTable table, final String catCol, final String[] numCol, final ExecutionContext exec) throws CanceledExecutionException, InvalidSettingsException {
    DataTableSpec spec = table.getSpec();
    int catColIdx = spec.findColumnIndex(catCol);
    int[] numColIdxs = new int[numCol.length];
    for (int i = 0; i < numCol.length; i++) {
        numColIdxs[i] = spec.findColumnIndex(numCol[i]);
    }
    Set<DataCell> valuesSet = spec.getColumnSpec(catColIdx).getDomain().getValues();
    if (valuesSet == null) {
        throw new InvalidSettingsException("Selected category column has no domain values");
    }
    ArrayList<DataCell> vals = new ArrayList<>(valuesSet);
    Collections.sort(vals, new Comparator<DataCell>() {

        @Override
        public int compare(final DataCell o1, final DataCell o2) {
            return o1.toString().compareTo(o2.toString());
        }
    });
    // add Missing values class as it is never in specification
    vals.add(new MissingCell(null));
    // we need to have clear names, otherwise Missing values class will be taken as "?"
    ArrayList<String> catNames = new ArrayList<>(vals.size());
    for (DataCell cell : vals) {
        catNames.add(cell.isMissing() ? MISSING_VALUES_CLASS : cell.toString());
    }
    LinkedHashMap<String, LinkedHashMap<String, DataContainer>> containers = new LinkedHashMap<>();
    m_ignoredMissVals = new LinkedHashMap<>();
    for (int i = 0; i < numCol.length; i++) {
        LinkedHashMap<String, DataContainer> map = new LinkedHashMap<>();
        LinkedHashMap<String, Long> missValMap = new LinkedHashMap<>();
        for (DataCell c : vals) {
            String name = c.isMissing() ? MISSING_VALUES_CLASS : c.toString();
            map.put(name, exec.createDataContainer(new DataTableSpec(new String[] { "col" }, new DataType[] { DoubleCell.TYPE })));
            missValMap.put(name, 0L);
        }
        containers.put(numCol[i], map);
        m_ignoredMissVals.put(numCol[i], missValMap);
    }
    ExecutionContext subExec = exec.createSubExecutionContext(0.7);
    // long[][] ignoredMissVals = new long[numCol.length][vals.size()];  // count missing values per data col per class
    long count = 0;
    final long numOfRows = table.size();
    for (DataRow row : table) {
        exec.checkCanceled();
        subExec.setProgress(count++ / (double) numOfRows);
        DataCell catCell = row.getCell(catColIdx);
        String catName = catCell.isMissing() ? MISSING_VALUES_CLASS : catCell.toString();
        for (int i = 0; i < numCol.length; i++) {
            DataCell cell = row.getCell(numColIdxs[i]);
            if (!cell.isMissing()) {
                containers.get(numCol[i]).get(catName).addRowToTable(new DefaultRow(row.getKey(), cell));
            } else {
                // increment missing values
                LinkedHashMap<String, Long> missValMap = m_ignoredMissVals.get(numCol[i]);
                missValMap.replace(catName, missValMap.get(catName) + 1);
            }
        }
    }
    LinkedHashMap<String, LinkedHashMap<String, BoxplotStatistics>> statsMap = new LinkedHashMap<>();
    excludedClasses = new LinkedHashMap<>();
    List<String> colList = Arrays.asList(numCol);
    ExecutionContext subExec2 = exec.createSubExecutionContext(1.0);
    int count2 = 0;
    for (Entry<String, LinkedHashMap<String, DataContainer>> entry : containers.entrySet()) {
        exec.checkCanceled();
        subExec2.setProgress(count2++ / (double) containers.size());
        LinkedHashMap<String, DataContainer> containers2 = entry.getValue();
        LinkedHashMap<String, BoxplotStatistics> colStats = new LinkedHashMap<String, BoxplotStatistics>();
        String colName = entry.getKey();
        List<String> excludedColClassesList = new ArrayList<>();
        LinkedHashMap<String, Long> ignoredColMissVals = new LinkedHashMap<>();
        for (Entry<String, DataContainer> entry2 : containers2.entrySet()) {
            Set<Outlier> extremeOutliers = new HashSet<Outlier>();
            Set<Outlier> mildOutliers = new HashSet<Outlier>();
            entry2.getValue().close();
            String catName = entry2.getKey();
            BufferedDataTable catTable = (BufferedDataTable) entry2.getValue().getTable();
            LinkedHashMap<String, Long> missValMap = m_ignoredMissVals.get(colName);
            if (catTable.size() == 0) {
                if (!(catName.equals(MISSING_VALUES_CLASS) && missValMap.get(catName) == 0)) {
                    // we should add missing values to this list, only if they were there
                    excludedColClassesList.add(catName);
                }
                missValMap.remove(catName);
                continue;
            } else {
                if (missValMap.get(catName) == 0) {
                    missValMap.remove(catName);
                }
            }
            SortedTable st = new SortedTable(catTable, new Comparator<DataRow>() {

                @Override
                public int compare(final DataRow o1, final DataRow o2) {
                    double d1 = ((DoubleValue) o1.getCell(0)).getDoubleValue();
                    double d2 = ((DoubleValue) o2.getCell(0)).getDoubleValue();
                    if (d1 == d2) {
                        return 0;
                    } else {
                        return d1 < d2 ? -1 : 1;
                    }
                }
            }, false, exec);
            double min = 0, max = 0, q1 = 0, q3 = 0, median = 0;
            boolean dq1 = catTable.size() % 4 == 0;
            long q1Idx = catTable.size() / 4;
            boolean dq3 = 3 * catTable.size() % 4 == 0;
            long q3Idx = 3 * catTable.size() / 4;
            boolean dMedian = catTable.size() % 2 == 0;
            long medianIdx = catTable.size() / 2;
            int counter = 0;
            for (DataRow row : st) {
                double val = ((DoubleValue) row.getCell(0)).getDoubleValue();
                if (counter == 0) {
                    min = val;
                }
                if (counter == catTable.size() - 1) {
                    max = val;
                }
                if (counter == q1Idx - 1 && dq1) {
                    q1 = val;
                }
                if (counter == q1Idx || (counter == 0 && st.size() <= 3)) {
                    if (dq1) {
                        q1 = (q1 + val) / 2.0;
                    } else {
                        q1 = val;
                    }
                }
                if (counter == medianIdx - 1 && dMedian) {
                    median = val;
                }
                if (counter == medianIdx) {
                    if (dMedian) {
                        median = (median + val) / 2;
                    } else {
                        median = val;
                    }
                }
                if (counter == q3Idx - 1 && dq3) {
                    q3 = val;
                }
                if (counter == q3Idx || (counter == st.size() - 1 && st.size() <= 3)) {
                    if (dq3) {
                        q3 = (q3 + val) / 2.0;
                    } else {
                        q3 = val;
                    }
                }
                counter++;
            }
            double iqr = q3 - q1;
            double lowerWhisker = min;
            double upperWhisker = max;
            double upperWhiskerFence = q3 + (1.5 * iqr);
            double lowerWhiskerFence = q1 - (1.5 * iqr);
            double lowerFence = q1 - (3 * iqr);
            double upperFence = q3 + (3 * iqr);
            for (DataRow row : st) {
                double value = ((DoubleValue) row.getCell(0)).getDoubleValue();
                String rowKey = row.getKey().getString();
                if (value < lowerFence) {
                    extremeOutliers.add(new Outlier(value, rowKey));
                } else if (value < lowerWhiskerFence) {
                    mildOutliers.add(new Outlier(value, rowKey));
                } else if (lowerWhisker < lowerWhiskerFence && value >= lowerWhiskerFence) {
                    lowerWhisker = value;
                } else if (value <= upperWhiskerFence) {
                    upperWhisker = value;
                } else if (value > upperFence) {
                    extremeOutliers.add(new Outlier(value, rowKey));
                } else if (value > upperWhiskerFence) {
                    mildOutliers.add(new Outlier(value, rowKey));
                }
            }
            colStats.put(catName, new BoxplotStatistics(mildOutliers, extremeOutliers, min, max, lowerWhisker, q1, median, q3, upperWhisker));
        }
        statsMap.put(colName, colStats);
        // missing values part
        String[] excludedColClasses = excludedColClassesList.toArray(new String[excludedColClassesList.size()]);
        excludedClasses.put(colName, excludedColClasses);
    }
    return statsMap;
}