Examples with PredictionProvider org.kie.kogito.explainability.model.PredictionProvider used on opensource projects

Example 46 with PredictionProvider

use of org.kie.kogito.explainability.model.PredictionProvider in project kogito-apps by kiegroup.

the class FairnessMetricsTest method testGroupDIRTextClassifier.

@Test
void testGroupDIRTextClassifier() throws ExecutionException, InterruptedException {
    List<PredictionInput> testInputs = getTestInputs();
    PredictionProvider model = TestUtils.getDummyTextClassifier();
    Predicate<PredictionInput> selector = predictionInput -> DataUtils.textify(predictionInput).contains("please");
    Output output = new Output("spam", Type.BOOLEAN, new Value(false), 1.0);
    double dir = FairnessMetrics.groupDisparateImpactRatio(selector, testInputs, model, output);
    assertThat(dir).isPositive();
}

Example 47 with PredictionProvider

use of org.kie.kogito.explainability.model.PredictionProvider in project kogito-apps by kiegroup.

the class FairnessMetricsTest method testIndividualConsistencyTextClassifier.

@Test
void testIndividualConsistencyTextClassifier() throws ExecutionException, InterruptedException {
    BiFunction<PredictionInput, List<PredictionInput>, List<PredictionInput>> proximityFunction = (predictionInput, predictionInputs) -> {
        String reference = DataUtils.textify(predictionInput);
        return predictionInputs.stream().sorted((o1, o2) -> (StringUtils.getFuzzyDistance(DataUtils.textify(o2), reference, Locale.getDefault()) - StringUtils.getFuzzyDistance(DataUtils.textify(o1), reference, Locale.getDefault()))).collect(Collectors.toList()).subList(1, 3);
    };
    List<PredictionInput> testInputs = getTestInputs();
    PredictionProvider model = TestUtils.getDummyTextClassifier();
    double individualConsistency = FairnessMetrics.individualConsistency(proximityFunction, testInputs, model);
    assertThat(individualConsistency).isBetween(0d, 1d);
}

Example 48 with PredictionProvider

use of org.kie.kogito.explainability.model.PredictionProvider in project kogito-apps by kiegroup.

the class ComplexEligibilityDmnCounterfactualExplainerTest method testDMNScoringFunction.

@Test
void testDMNScoringFunction() throws ExecutionException, InterruptedException, TimeoutException {
    PredictionProvider model = getModel();
    final List<Output> goal = generateGoal(true, true, 1.0);
    List<Feature> features = new LinkedList<>();
    features.add(FeatureFactory.newNumericalFeature("age", 40, NumericalFeatureDomain.create(18, 60)));
    features.add(FeatureFactory.newBooleanFeature("hasReferral", true));
    features.add(FeatureFactory.newNumericalFeature("monthlySalary", 500, NumericalFeatureDomain.create(10, 100_000)));
    final TerminationConfig terminationConfig = new TerminationConfig().withScoreCalculationCountLimit(10_000L);
    // for the purpose of this test, only a few steps are necessary
    final SolverConfig solverConfig = SolverConfigBuilder.builder().withTerminationConfig(terminationConfig).build();
    solverConfig.setRandomSeed((long) 23);
    solverConfig.setEnvironmentMode(EnvironmentMode.REPRODUCIBLE);
    final CounterfactualConfig counterfactualConfig = new CounterfactualConfig().withSolverConfig(solverConfig).withGoalThreshold(0.01);
    final CounterfactualExplainer counterfactualExplainer = new CounterfactualExplainer(counterfactualConfig);
    PredictionInput input = new PredictionInput(features);
    PredictionOutput output = new PredictionOutput(goal);
    Prediction prediction = new CounterfactualPrediction(input, output, null, UUID.randomUUID(), 60L);
    final CounterfactualResult counterfactualResult = counterfactualExplainer.explainAsync(prediction, model).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    List<Output> cfOutputs = counterfactualResult.getOutput().get(0).getOutputs();
    assertTrue(counterfactualResult.isValid());
    assertEquals("inputsAreValid", cfOutputs.get(0).getName());
    assertTrue((Boolean) cfOutputs.get(0).getValue().getUnderlyingObject());
    assertEquals("canRequestLoan", cfOutputs.get(1).getName());
    assertTrue((Boolean) cfOutputs.get(1).getValue().getUnderlyingObject());
    assertEquals("my-scoring-function", cfOutputs.get(2).getName());
    assertEquals(1.0, ((BigDecimal) cfOutputs.get(2).getValue().getUnderlyingObject()).doubleValue(), 0.01);
    List<CounterfactualEntity> entities = counterfactualResult.getEntities();
    assertEquals("age", entities.get(0).asFeature().getName());
    assertEquals(18, entities.get(0).asFeature().getValue().asNumber());
    assertEquals("hasReferral", entities.get(1).asFeature().getName());
    assertTrue((Boolean) entities.get(1).asFeature().getValue().getUnderlyingObject());
    assertEquals("monthlySalary", entities.get(2).asFeature().getName());
    final double monthlySalary = entities.get(2).asFeature().getValue().asNumber();
    assertEquals(7900, monthlySalary, 10);
    // since the scoring function is ((0.6 * ((42 - age + 18)/42)) + (0.4 * (monthlySalary/8000)))
    // for a result of 1.0 the relation must be age = (7*monthlySalary)/2000 - 10
    assertEquals(18, (7 * monthlySalary) / 2000.0 - 10.0, 0.5);
}

Example 49 with PredictionProvider

use of org.kie.kogito.explainability.model.PredictionProvider in project kogito-apps by kiegroup.

the class ComplexEligibilityDmnCounterfactualExplainerTest method testDMNInvalidCounterfactualExplanation.

@Test
void testDMNInvalidCounterfactualExplanation() throws ExecutionException, InterruptedException, TimeoutException {
    PredictionProvider model = getModel();
    final List<Output> goal = generateGoal(true, true, 0.6);
    List<Feature> features = new LinkedList<>();
    // DMN model does not allow loans for age >= 60, so no CF will be possible
    features.add(FeatureFactory.newNumericalFeature("age", 61));
    features.add(FeatureFactory.newBooleanFeature("hasReferral", true));
    features.add(FeatureFactory.newNumericalFeature("monthlySalary", 500, NumericalFeatureDomain.create(10, 10_000)));
    final TerminationConfig terminationConfig = new TerminationConfig().withScoreCalculationCountLimit(10_000L);
    // for the purpose of this test, only a few steps are necessary
    final SolverConfig solverConfig = SolverConfigBuilder.builder().withTerminationConfig(terminationConfig).build();
    solverConfig.setRandomSeed((long) 23);
    solverConfig.setEnvironmentMode(EnvironmentMode.REPRODUCIBLE);
    final CounterfactualConfig counterfactualConfig = new CounterfactualConfig().withSolverConfig(solverConfig).withGoalThreshold(0.01);
    final CounterfactualExplainer counterfactualExplainer = new CounterfactualExplainer(counterfactualConfig);
    PredictionInput input = new PredictionInput(features);
    PredictionOutput output = new PredictionOutput(goal);
    Prediction prediction = new CounterfactualPrediction(input, output, null, UUID.randomUUID(), 60L);
    final CounterfactualResult counterfactualResult = counterfactualExplainer.explainAsync(prediction, model).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    assertFalse(counterfactualResult.isValid());
}

Example 50 with PredictionProvider

use of org.kie.kogito.explainability.model.PredictionProvider in project kogito-apps by kiegroup.

the class ComplexEligibilityDmnCounterfactualExplainerTest method testDMNValidCounterfactualExplanation.

@Test
void testDMNValidCounterfactualExplanation() throws ExecutionException, InterruptedException, TimeoutException {
    PredictionProvider model = getModel();
    final List<Output> goal = generateGoal(true, true, 0.6);
    List<Feature> features = new LinkedList<>();
    features.add(FeatureFactory.newNumericalFeature("age", 40));
    features.add(FeatureFactory.newBooleanFeature("hasReferral", true));
    features.add(FeatureFactory.newNumericalFeature("monthlySalary", 500, NumericalFeatureDomain.create(10, 10_000)));
    final TerminationConfig terminationConfig = new TerminationConfig().withScoreCalculationCountLimit(10_000L);
    // for the purpose of this test, only a few steps are necessary
    final SolverConfig solverConfig = SolverConfigBuilder.builder().withTerminationConfig(terminationConfig).build();
    solverConfig.setRandomSeed((long) 23);
    solverConfig.setEnvironmentMode(EnvironmentMode.REPRODUCIBLE);
    final CounterfactualConfig counterfactualConfig = new CounterfactualConfig().withSolverConfig(solverConfig).withGoalThreshold(0.01);
    final CounterfactualExplainer counterfactualExplainer = new CounterfactualExplainer(counterfactualConfig);
    PredictionInput input = new PredictionInput(features);
    PredictionOutput output = new PredictionOutput(goal);
    Prediction prediction = new CounterfactualPrediction(input, output, null, UUID.randomUUID(), 60L);
    final CounterfactualResult counterfactualResult = counterfactualExplainer.explainAsync(prediction, model).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    List<Output> cfOutputs = counterfactualResult.getOutput().get(0).getOutputs();
    assertTrue(counterfactualResult.isValid());
    assertEquals("inputsAreValid", cfOutputs.get(0).getName());
    assertTrue((Boolean) cfOutputs.get(0).getValue().getUnderlyingObject());
    assertEquals("canRequestLoan", cfOutputs.get(1).getName());
    assertTrue((Boolean) cfOutputs.get(1).getValue().getUnderlyingObject());
    assertEquals("my-scoring-function", cfOutputs.get(2).getName());
    assertEquals(0.6, ((BigDecimal) cfOutputs.get(2).getValue().getUnderlyingObject()).doubleValue(), 0.05);
    List<CounterfactualEntity> entities = counterfactualResult.getEntities();
    assertEquals("age", entities.get(0).asFeature().getName());
    assertEquals(40, entities.get(0).asFeature().getValue().asNumber());
    assertEquals("hasReferral", entities.get(1).asFeature().getName());
    assertTrue((Boolean) entities.get(1).asFeature().getValue().getUnderlyingObject());
    assertEquals("monthlySalary", entities.get(2).asFeature().getName());
    assertTrue(entities.get(2).asFeature().getValue().asNumber() > 6000);
}