Examples with DescriptiveStatistics org.apache.commons.math3.stat.descriptive.DescriptiveStatistics used on opensource projects

Example 46 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project qupath by qupath.

the class CellIntensityClassificationCommand method run.

@Override
public void run() {
    // Dialogs.showErrorNotification(title, "Not implemented yet!");
    var imageData = qupath.getImageData();
    if (imageData == null) {
        Dialogs.showNoImageError(title);
        return;
    }
    var hierarchy = imageData.getHierarchy();
    // Try to operate on cells, but accept operating on all detections if necessary
    var cells = imageData.getHierarchy().getCellObjects();
    boolean allDetections = cells.isEmpty();
    if (allDetections)
        logger.debug("No cells found - will try using all detections");
    var detections = allDetections ? imageData.getHierarchy().getDetectionObjects() : cells;
    if (detections.isEmpty()) {
        Dialogs.showErrorMessage(title, "No cells found in the current image!");
        return;
    }
    var measurements = PathClassifierTools.getAvailableFeatures(detections);
    if (measurements.isEmpty()) {
        Dialogs.showErrorMessage(title, "No cell measurements found in the current image!");
        return;
    }
    var currentClassifications = PathClassifierTools.createClassificationMap(detections);
    var comboMeasurements = new ComboBox<String>();
    comboMeasurements.getItems().setAll(measurements);
    PaneTools.setToExpandGridPaneWidth(comboMeasurements);
    var selectedMeasurement = comboMeasurements.getSelectionModel().selectedItemProperty();
    var cbSingleThreshold = new CheckBox("Single threshold");
    cbSingleThreshold.setSelected(true);
    var singleThreshold = cbSingleThreshold.selectedProperty();
    var sliders = new ArrayList<Slider>();
    var textFields = new ArrayList<TextField>();
    for (int i = 0; i < 3; i++) {
        var slider = new Slider();
        var tf = new TextField();
        tf.setPrefColumnCount(6);
        textFields.add(tf);
        GuiTools.bindSliderAndTextField(slider, tf, true);
        GuiTools.installRangePrompt(slider);
        slider.valueProperty().addListener((v, o, n) -> {
            updateClassifications(hierarchy, allDetections, selectedMeasurement.get(), parseValues(sliders, singleThreshold.get()));
        });
        PaneTools.setToExpandGridPaneWidth(slider);
        sliders.add(slider);
    }
    var map = new HashMap<String, double[]>();
    var histogramPanel = new HistogramPanelFX();
    var chartWrapper = new ThresholdedChartWrapper(histogramPanel.getChart());
    chartWrapper.setIsInteractive(true);
    singleThreshold.addListener((v, o, n) -> {
        chartWrapper.clearThresholds();
        Color color = Color.rgb(0, 0, 0, 0.2);
        if (!n) {
            for (int i = 0; i < sliders.size(); i++) {
                chartWrapper.addThreshold(sliders.get(i).valueProperty(), color);
            }
        } else
            chartWrapper.addThreshold(sliders.get(0).valueProperty(), color);
    });
    selectedMeasurement.addListener((v, o, n) -> {
        if (o != null)
            map.put(o, parseValues(sliders));
        double[] measurementValues = detections.stream().mapToDouble(p -> p.getMeasurementList().getMeasurementValue(n)).filter(d -> Double.isFinite(d)).toArray();
        var stats = new DescriptiveStatistics(measurementValues);
        var histogram = new Histogram(measurementValues, 100, stats.getMin(), stats.getMax());
        histogramPanel.getHistogramData().setAll(HistogramPanelFX.createHistogramData(histogram, false, ColorTools.packARGB(100, 200, 20, 20)));
        double[] values = map.get(n);
        for (int i = 0; i < sliders.size(); i++) {
            var slider = sliders.get(i);
            slider.setMin(stats.getMin());
            slider.setMax(stats.getMax());
            double val = values == null ? stats.getMean() + stats.getStandardDeviation() * i : values[i];
            slider.setValue(val);
            // Add first threshold to histogram
            if (i == 0) {
                Color color = Color.rgb(0, 0, 0, 0.2);
                chartWrapper.addThreshold(sliders.get(i).valueProperty(), color);
            }
        }
    });
    selectedMeasurement.addListener((v, o, n) -> updateClassifications(hierarchy, allDetections, n, parseValues(sliders, singleThreshold.get())));
    singleThreshold.addListener((v, o, n) -> updateClassifications(hierarchy, allDetections, selectedMeasurement.get(), parseValues(sliders, singleThreshold.get())));
    var pane = new GridPane();
    int row = 0;
    var labelMeasurements = new Label("Measurement");
    PaneTools.addGridRow(pane, row++, 0, "Select measurement to threshold", labelMeasurements, comboMeasurements, comboMeasurements);
    for (int i = 0; i < sliders.size(); i++) {
        var labelThreshold = new Label("Threshold " + (i + 1) + "+");
        var slider = sliders.get(i);
        var tf = textFields.get(i);
        if (i > 0) {
            slider.disableProperty().bind(singleThreshold);
            tf.disableProperty().bind(singleThreshold);
        }
        PaneTools.addGridRow(pane, row++, 0, "Select threshold value", labelThreshold, slider, tf);
    }
    PaneTools.addGridRow(pane, row++, 0, "Toggle between using a single threshold (Negative/Positive) or three threshold Negative/1+/2+/3+)", cbSingleThreshold, cbSingleThreshold, cbSingleThreshold);
    pane.setHgap(5.0);
    pane.setVgap(5.0);
    PaneTools.setToExpandGridPaneHeight(chartWrapper.getPane());
    PaneTools.setToExpandGridPaneWidth(chartWrapper.getPane());
    histogramPanel.getChart().getYAxis().setTickLabelsVisible(false);
    histogramPanel.getChart().setAnimated(false);
    chartWrapper.getPane().setPrefSize(200, 80);
    pane.add(chartWrapper.getPane(), pane.getColumnCount(), 0, 1, pane.getRowCount());
    var dialog = new Dialog<ButtonType>();
    dialog.initOwner(qupath.getStage());
    dialog.setTitle(title);
    dialog.getDialogPane().setContent(pane);
    dialog.getDialogPane().getButtonTypes().setAll(ButtonType.APPLY, ButtonType.CANCEL);
    var response = dialog.showAndWait().orElse(ButtonType.CANCEL);
    if (pool != null) {
        pool.shutdown();
        try {
            pool.awaitTermination(5000L, TimeUnit.SECONDS);
        } catch (InterruptedException e) {
            logger.debug("Exception waiting for classification to complete: " + e.getLocalizedMessage(), e);
        }
    }
    // Check if we did anything, if not return
    if (nextRequest == null)
        return;
    if (ButtonType.APPLY.equals(response)) {
        // Make sure we ran the last command, then log it in the workflow
        if (!nextRequest.isComplete())
            nextRequest.doClassification();
        imageData.getHistoryWorkflow().addStep(nextRequest.toWorkflowStep());
    } else {
        // Restore classifications if the user cancelled
        var changed = PathClassifierTools.restoreClassificationsFromMap(currentClassifications);
        if (!changed.isEmpty())
            hierarchy.fireObjectClassificationsChangedEvent(this, changed);
    }
}

Example 47 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project urban-eureka by errir503.

the class AbstractTestApproximateCountDistinct method testMultiplePositions.

@Test(dataProvider = "provideStandardErrors")
public void testMultiplePositions(double maxStandardError) {
    DescriptiveStatistics stats = new DescriptiveStatistics();
    for (int i = 0; i < 500; ++i) {
        int uniques = ThreadLocalRandom.current().nextInt(getUniqueValuesCount()) + 1;
        List<Object> values = createRandomSample(uniques, (int) (uniques * 1.5));
        long actual = estimateGroupByCount(values, maxStandardError);
        double error = (actual - uniques) * 1.0 / uniques;
        stats.addValue(error);
    }
    assertLessThan(stats.getMean(), 1.0e-2);
    assertLessThan(stats.getStandardDeviation(), 1.0e-2 + maxStandardError);
}

Example 48 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project concourse by cinchapi.

the class TrackingMultimap method spread.

/**
 * Return a relative measure of the statistical dispersion in this data.
 * <p>
 * There are several ways to measure statistical dispersion, so callers
 * should not rely on a specific underlying implementation because it may
 * change over time. This method simply offers a value that allows for
 * comparison of dispersion across data sets.
 * </p>
 * <p>
 * A larger dispersion value means that the data is more spread out whereas
 * a smaller dispersion value indicates the opposite.
 * </p>
 *
 * @return the dispersion value for this data
 */
public double spread() {
    // Get the quartile coefficient of dispersion, which is a cross
    // dataset mechanism for comparing the relative dispersion of data.
    double[] frequencies = new double[size()];
    AtomicInteger index = new AtomicInteger(0);
    data.values().forEach(records -> frequencies[index.getAndIncrement()] = records.size());
    DescriptiveStatistics stats = new DescriptiveStatistics(frequencies);
    double p1 = stats.getPercentile(25);
    double p3 = stats.getPercentile(75);
    double coefficientOfDispersion = (p3 - p1) / (p3 + p1);
    // Grab the coefficient of variance
    double coefficientOfVariance = stats.getStandardDeviation() / stats.getMean();
    // Calculate the average absolute deviation from the mean
    double[] deviations = new double[frequencies.length];
    for (int i = 0; i < deviations.length; ++i) {
        deviations[i] = Math.abs(frequencies[i] - stats.getMean());
    }
    double averageAbsoluteDeviation = StatUtils.mean(deviations) / stats.getMean();
    // Apply a weighting to the various components
    return (0.50 * coefficientOfDispersion) + (0.40 * coefficientOfVariance) + (0.10 * averageAbsoluteDeviation);
}

Example 49 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project hive by apache.

the class ReplStatsTracker method addEntry.

/**
 * Adds an entry for tracking.
 * @param eventType the type of event.
 * @param eventId the event id.
 * @param timeTaken time taken to process the event.
 */
public synchronized void addEntry(String eventType, String eventId, long timeTaken) {
    // Store the last EventId for the JMX.
    lastEventId = eventId;
    // Update the entry in the descriptive statistics.
    DescriptiveStatistics descStatistics = descMap.get(eventType);
    if (descStatistics == null) {
        descStatistics = new DescriptiveStatistics();
        descStatistics.addValue(timeTaken);
        descMap.put(eventType, descStatistics);
    } else {
        descStatistics.addValue(timeTaken);
    }
    // Tracking for top K events, Maintain the list in descending order.
    ListOrderedMap<Long, Long> topKEntries = topKEvents.get(eventType);
    if (topKEntries == null) {
        topKEntries = new ListOrderedMap<>();
        topKEntries.put(Long.parseLong(eventId), timeTaken);
        topKEvents.put(eventType, topKEntries);
    } else {
        // Get the index of insertion, by descending order.
        int index = Collections.binarySearch(new ArrayList(topKEntries.values()), timeTaken, Collections.reverseOrder());
        // If the element comes as top K add it to the topEntries.
        // The index returned from the binary search, is either the index where the element already exist, else
        // (-insertionIndex) -1, so convert it to actual insertion index
        int insertionIndex = index < 0 ? -1 * (index) - 1 : index;
        if (insertionIndex < k && k >= 0) {
            topKEntries.put(insertionIndex, Long.parseLong(eventId), timeTaken);
        }
        // entries, since the list is sorted in descending order.
        if (topKEntries.size() > k) {
            topKEntries.remove(k);
        }
    }
}

Example 50 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project hive by apache.

the class HMSBenchmarks method benchmarkRenameTable.

static DescriptiveStatistics benchmarkRenameTable(@NotNull MicroBenchmark bench, @NotNull BenchData data, int count) {
    final HMSClient client = data.getClient();
    String dbName = data.dbName;
    String tableName = data.tableName;
    BenchmarkUtils.createPartitionedTable(client, dbName, tableName);
    try {
        addManyPartitionsNoException(client, dbName, tableName, null, Collections.singletonList("d"), count);
        Table oldTable = client.getTable(dbName, tableName);
        oldTable.getSd().setLocation("");
        Table newTable = oldTable.deepCopy();
        newTable.setTableName(tableName + "_renamed");
        return bench.measure(() -> {
            // Measuring 2 renames, so the tests are idempotent
            throwingSupplierWrapper(() -> client.alterTable(oldTable.getDbName(), oldTable.getTableName(), newTable));
            throwingSupplierWrapper(() -> client.alterTable(newTable.getDbName(), newTable.getTableName(), oldTable));
        });
    } catch (TException e) {
        e.printStackTrace();
        return new DescriptiveStatistics();
    } finally {
        throwingSupplierWrapper(() -> client.dropTable(dbName, tableName));
    }
}