Examples with Dataset org.kie.kogito.explainability.model.Dataset used on opensource projects

Example 1 with Dataset

use of org.kie.kogito.explainability.model.Dataset in project kogito-apps by kiegroup.

the class FairnessMetrics method groupAverageOddsDifference.

/**
 * Calculate average odds difference.
 *
 * @param inputSelector selector for privileged group
 * @param outputSelector selector for favorable label
 * @param dataset dataset used to evaluate AOD
 * @param model model to be evaluated fairness-wise
 * @return average odds difference value
 * @throws ExecutionException if any error occurs during model prediction
 * @throws InterruptedException if timeout or other interruption issues occur during model prediction
 */
public static double groupAverageOddsDifference(Predicate<PredictionInput> inputSelector, Predicate<PredictionOutput> outputSelector, Dataset dataset, PredictionProvider model) throws ExecutionException, InterruptedException {
    Dataset privileged = dataset.filterByInput(inputSelector);
    Map<String, Integer> privilegedCounts = countMatchingOutputSelector(privileged, model.predictAsync(privileged.getInputs()).get(), outputSelector);
    Dataset unprivileged = dataset.filterByInput(inputSelector.negate());
    Map<String, Integer> unprivilegedCounts = countMatchingOutputSelector(unprivileged, model.predictAsync(unprivileged.getInputs()).get(), outputSelector);
    double utp = unprivilegedCounts.get("tp");
    double utn = unprivilegedCounts.get("tn");
    double ufp = unprivilegedCounts.get("fp");
    double ufn = unprivilegedCounts.get("fn");
    double ptp = privilegedCounts.get("tp");
    double ptn = privilegedCounts.get("tn");
    double pfp = privilegedCounts.get("fp");
    double pfn = privilegedCounts.get("fn");
    return (utp / (utp + ufn) - ptp / (ptp + pfn + 1e-10)) / 2d + (ufp / (ufp + utn) - pfp / (pfp + ptn + 1e-10)) / 2;
}

Example 2 with Dataset

use of org.kie.kogito.explainability.model.Dataset in project kogito-apps by kiegroup.

the class FairnessMetrics method groupAveragePredictiveValueDifference.

/**
 * Calculate average predictive value difference.
 *
 * @param inputSelector selector for privileged group
 * @param outputSelector selector for favorable label
 * @param dataset dataset used to evaluate AOD
 * @param model model to be evaluated fairness-wise
 * @return average predictive value difference
 * @throws ExecutionException if any error occurs during model prediction
 * @throws InterruptedException if timeout or other interruption issues occur during model prediction
 */
public static double groupAveragePredictiveValueDifference(Predicate<PredictionInput> inputSelector, Predicate<PredictionOutput> outputSelector, Dataset dataset, PredictionProvider model) throws ExecutionException, InterruptedException {
    Dataset privileged = dataset.filterByInput(inputSelector);
    Map<String, Integer> privilegedCounts = countMatchingOutputSelector(privileged, model.predictAsync(privileged.getInputs()).get(), outputSelector);
    double ptp = privilegedCounts.get("tp");
    double ptn = privilegedCounts.get("tn");
    double pfp = privilegedCounts.get("fp");
    double pfn = privilegedCounts.get("fn");
    Dataset unprivileged = dataset.filterByInput(inputSelector.negate());
    Map<String, Integer> unprivilegedCounts = countMatchingOutputSelector(unprivileged, model.predictAsync(unprivileged.getInputs()).get(), outputSelector);
    double utp = unprivilegedCounts.get("tp");
    double utn = unprivilegedCounts.get("tn");
    double ufp = unprivilegedCounts.get("fp");
    double ufn = unprivilegedCounts.get("fn");
    return (utp / (utp + ufp) - ptp / (ptp + pfp + 1e-10)) / 2d + (ufn / (ufn + utn) - pfn / (pfn + ptn + 1e-10)) / 2;
}

Example 3 with Dataset

use of org.kie.kogito.explainability.model.Dataset in project kogito-apps by kiegroup.

the class FairnessMetricsTest method testGroupAPVDTextClassifier.

@Test
void testGroupAPVDTextClassifier() throws ExecutionException, InterruptedException {
    List<Prediction> predictions = getTestData();
    Dataset dataset = new Dataset(predictions);
    PredictionProvider model = TestUtils.getDummyTextClassifier();
    Predicate<PredictionInput> inputSelector = predictionInput -> DataUtils.textify(predictionInput).contains("please");
    Predicate<PredictionOutput> outputSelector = predictionOutput -> predictionOutput.getByName("spam").get().getValue().asNumber() == 0;
    double apvd = FairnessMetrics.groupAveragePredictiveValueDifference(inputSelector, outputSelector, dataset, model);
    assertThat(apvd).isBetween(-1d, 1d);
}

Example 4 with Dataset

use of org.kie.kogito.explainability.model.Dataset in project kogito-apps by kiegroup.

the class FairnessMetricsTest method testGroupAODTextClassifier.

@Test
void testGroupAODTextClassifier() throws ExecutionException, InterruptedException {
    List<Prediction> predictions = getTestData();
    Dataset dataset = new Dataset(predictions);
    PredictionProvider model = TestUtils.getDummyTextClassifier();
    Predicate<PredictionInput> inputSelector = predictionInput -> DataUtils.textify(predictionInput).contains("please");
    Predicate<PredictionOutput> outputSelector = predictionOutput -> predictionOutput.getByName("spam").get().getValue().asNumber() == 0;
    double aod = FairnessMetrics.groupAverageOddsDifference(inputSelector, outputSelector, dataset, model);
    assertThat(aod).isBetween(-1d, 1d);
}