Examples with Prediction org.kie.kogito.explainability.model.Prediction used on opensource projects

Example 41 with Prediction

use of org.kie.kogito.explainability.model.Prediction in project kogito-apps by kiegroup.

the class PmmlRegressionCategoricalLimeExplainerTest method testExplanationStabilityWithOptimization.

@Disabled("See KOGITO-6154")
@Test
void testExplanationStabilityWithOptimization() throws ExecutionException, InterruptedException, TimeoutException {
    PredictionProvider model = getModel();
    List<PredictionInput> samples = getSamples();
    List<PredictionOutput> predictionOutputs = model.predictAsync(samples.subList(0, 5)).get();
    List<Prediction> predictions = DataUtils.getPredictions(samples, predictionOutputs);
    long seed = 0;
    LimeConfigOptimizer limeConfigOptimizer = new LimeConfigOptimizer().withDeterministicExecution(true);
    Random random = new Random();
    PerturbationContext perturbationContext = new PerturbationContext(seed, random, 1);
    LimeConfig initialConfig = new LimeConfig().withSamples(10).withPerturbationContext(perturbationContext);
    LimeConfig optimizedConfig = limeConfigOptimizer.optimize(initialConfig, predictions, model);
    assertThat(optimizedConfig).isNotSameAs(initialConfig);
    LimeExplainer limeExplainer = new LimeExplainer(optimizedConfig);
    PredictionInput testPredictionInput = getTestInput();
    List<PredictionOutput> testPredictionOutputs = model.predictAsync(List.of(testPredictionInput)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    Prediction instance = new SimplePrediction(testPredictionInput, testPredictionOutputs.get(0));
    assertDoesNotThrow(() -> ValidationUtils.validateLocalSaliencyStability(model, instance, limeExplainer, 1, 0.6, 0.6));
}

Example 42 with Prediction

use of org.kie.kogito.explainability.model.Prediction in project kogito-apps by kiegroup.

the class DummyModelsLimeExplainerTest method testUnusedFeatureClassification.

@ParameterizedTest
@ValueSource(longs = { 0 })
void testUnusedFeatureClassification(long seed) throws Exception {
    Random random = new Random();
    int idx = 2;
    List<Feature> features = new LinkedList<>();
    features.add(FeatureFactory.newNumericalFeature("f1", 6));
    features.add(FeatureFactory.newNumericalFeature("f2", 3));
    features.add(FeatureFactory.newNumericalFeature("f3", 5));
    PredictionProvider model = TestUtils.getEvenSumModel(idx);
    PredictionInput input = new PredictionInput(features);
    List<PredictionOutput> outputs = model.predictAsync(List.of(input)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    Prediction prediction = new SimplePrediction(input, outputs.get(0));
    LimeConfig limeConfig = new LimeConfig().withSamples(100).withPerturbationContext(new PerturbationContext(seed, random, 1));
    LimeExplainer limeExplainer = new LimeExplainer(limeConfig);
    Map<String, Saliency> saliencyMap = limeExplainer.explainAsync(prediction, model).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    for (Saliency saliency : saliencyMap.values()) {
        assertNotNull(saliency);
        List<FeatureImportance> topFeatures = saliency.getTopFeatures(3);
        assertEquals(3, topFeatures.size());
        assertEquals(1d, ExplainabilityMetrics.impactScore(model, prediction, topFeatures));
    }
    int topK = 1;
    double minimumPositiveStabilityRate = 0.5;
    double minimumNegativeStabilityRate = 0.5;
    TestUtils.assertLimeStability(model, prediction, limeExplainer, topK, minimumPositiveStabilityRate, minimumNegativeStabilityRate);
    List<PredictionInput> inputs = new ArrayList<>();
    for (int i = 0; i < 100; i++) {
        List<Feature> fs = new LinkedList<>();
        fs.add(TestUtils.getMockedNumericFeature());
        fs.add(TestUtils.getMockedNumericFeature());
        fs.add(TestUtils.getMockedNumericFeature());
        inputs.add(new PredictionInput(fs));
    }
    DataDistribution distribution = new PredictionInputsDataDistribution(inputs);
    int k = 2;
    int chunkSize = 10;
    String decision = "sum-even-but" + idx;
    double precision = ExplainabilityMetrics.getLocalSaliencyPrecision(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(precision).isEqualTo(1);
    double recall = ExplainabilityMetrics.getLocalSaliencyRecall(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(recall).isEqualTo(1);
    double f1 = ExplainabilityMetrics.getLocalSaliencyF1(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(f1).isEqualTo(1);
}

Example 43 with Prediction

use of org.kie.kogito.explainability.model.Prediction in project kogito-apps by kiegroup.

the class DummyModelsLimeExplainerTest method testTextSpamClassification.

@ParameterizedTest
@ValueSource(longs = { 0 })
void testTextSpamClassification(long seed) throws Exception {
    Random random = new Random();
    List<Feature> features = new LinkedList<>();
    Function<String, List<String>> tokenizer = s -> Arrays.asList(s.split(" ").clone());
    features.add(FeatureFactory.newFulltextFeature("f1", "we go here and there", tokenizer));
    features.add(FeatureFactory.newFulltextFeature("f2", "please give me some money", tokenizer));
    features.add(FeatureFactory.newFulltextFeature("f3", "dear friend, please reply", tokenizer));
    PredictionInput input = new PredictionInput(features);
    PredictionProvider model = TestUtils.getDummyTextClassifier();
    List<PredictionOutput> outputs = model.predictAsync(List.of(input)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    Prediction prediction = new SimplePrediction(input, outputs.get(0));
    LimeConfig limeConfig = new LimeConfig().withSamples(100).withPerturbationContext(new PerturbationContext(seed, random, 1));
    LimeExplainer limeExplainer = new LimeExplainer(limeConfig);
    Map<String, Saliency> saliencyMap = limeExplainer.explainAsync(prediction, model).toCompletableFuture().get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    for (Saliency saliency : saliencyMap.values()) {
        assertNotNull(saliency);
        List<FeatureImportance> topFeatures = saliency.getPositiveFeatures(1);
        assertEquals(1, topFeatures.size());
        assertEquals(1d, ExplainabilityMetrics.impactScore(model, prediction, topFeatures));
    }
    int topK = 1;
    double minimumPositiveStabilityRate = 0.5;
    double minimumNegativeStabilityRate = 0.2;
    TestUtils.assertLimeStability(model, prediction, limeExplainer, topK, minimumPositiveStabilityRate, minimumNegativeStabilityRate);
    List<PredictionInput> inputs = new ArrayList<>();
    for (int i = 0; i < 100; i++) {
        List<Feature> fs = new LinkedList<>();
        fs.add(TestUtils.getMockedNumericFeature());
        fs.add(TestUtils.getMockedNumericFeature());
        fs.add(TestUtils.getMockedNumericFeature());
        inputs.add(new PredictionInput(fs));
    }
    DataDistribution distribution = new PredictionInputsDataDistribution(inputs);
    int k = 2;
    int chunkSize = 10;
    String decision = "spam";
    double precision = ExplainabilityMetrics.getLocalSaliencyPrecision(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(precision).isEqualTo(1);
    double recall = ExplainabilityMetrics.getLocalSaliencyRecall(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(recall).isEqualTo(1);
    double f1 = ExplainabilityMetrics.getLocalSaliencyF1(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(f1).isEqualTo(1);
}

Example 44 with Prediction

use of org.kie.kogito.explainability.model.Prediction in project kogito-apps by kiegroup.

the class ComplexEligibilityDmnCounterfactualExplainerTest method testDMNScoringFunction.

@Test
void testDMNScoringFunction() throws ExecutionException, InterruptedException, TimeoutException {
    PredictionProvider model = getModel();
    final List<Output> goal = generateGoal(true, true, 1.0);
    List<Feature> features = new LinkedList<>();
    features.add(FeatureFactory.newNumericalFeature("age", 40, NumericalFeatureDomain.create(18, 60)));
    features.add(FeatureFactory.newBooleanFeature("hasReferral", true));
    features.add(FeatureFactory.newNumericalFeature("monthlySalary", 500, NumericalFeatureDomain.create(10, 100_000)));
    final TerminationConfig terminationConfig = new TerminationConfig().withScoreCalculationCountLimit(10_000L);
    // for the purpose of this test, only a few steps are necessary
    final SolverConfig solverConfig = SolverConfigBuilder.builder().withTerminationConfig(terminationConfig).build();
    solverConfig.setRandomSeed((long) 23);
    solverConfig.setEnvironmentMode(EnvironmentMode.REPRODUCIBLE);
    final CounterfactualConfig counterfactualConfig = new CounterfactualConfig().withSolverConfig(solverConfig).withGoalThreshold(0.01);
    final CounterfactualExplainer counterfactualExplainer = new CounterfactualExplainer(counterfactualConfig);
    PredictionInput input = new PredictionInput(features);
    PredictionOutput output = new PredictionOutput(goal);
    Prediction prediction = new CounterfactualPrediction(input, output, null, UUID.randomUUID(), 60L);
    final CounterfactualResult counterfactualResult = counterfactualExplainer.explainAsync(prediction, model).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    List<Output> cfOutputs = counterfactualResult.getOutput().get(0).getOutputs();
    assertTrue(counterfactualResult.isValid());
    assertEquals("inputsAreValid", cfOutputs.get(0).getName());
    assertTrue((Boolean) cfOutputs.get(0).getValue().getUnderlyingObject());
    assertEquals("canRequestLoan", cfOutputs.get(1).getName());
    assertTrue((Boolean) cfOutputs.get(1).getValue().getUnderlyingObject());
    assertEquals("my-scoring-function", cfOutputs.get(2).getName());
    assertEquals(1.0, ((BigDecimal) cfOutputs.get(2).getValue().getUnderlyingObject()).doubleValue(), 0.01);
    List<CounterfactualEntity> entities = counterfactualResult.getEntities();
    assertEquals("age", entities.get(0).asFeature().getName());
    assertEquals(18, entities.get(0).asFeature().getValue().asNumber());
    assertEquals("hasReferral", entities.get(1).asFeature().getName());
    assertTrue((Boolean) entities.get(1).asFeature().getValue().getUnderlyingObject());
    assertEquals("monthlySalary", entities.get(2).asFeature().getName());
    final double monthlySalary = entities.get(2).asFeature().getValue().asNumber();
    assertEquals(7900, monthlySalary, 10);
    // since the scoring function is ((0.6 * ((42 - age + 18)/42)) + (0.4 * (monthlySalary/8000)))
    // for a result of 1.0 the relation must be age = (7*monthlySalary)/2000 - 10
    assertEquals(18, (7 * monthlySalary) / 2000.0 - 10.0, 0.5);
}

Example 45 with Prediction

use of org.kie.kogito.explainability.model.Prediction in project kogito-apps by kiegroup.

the class ComplexEligibilityDmnCounterfactualExplainerTest method testDMNInvalidCounterfactualExplanation.

@Test
void testDMNInvalidCounterfactualExplanation() throws ExecutionException, InterruptedException, TimeoutException {
    PredictionProvider model = getModel();
    final List<Output> goal = generateGoal(true, true, 0.6);
    List<Feature> features = new LinkedList<>();
    // DMN model does not allow loans for age >= 60, so no CF will be possible
    features.add(FeatureFactory.newNumericalFeature("age", 61));
    features.add(FeatureFactory.newBooleanFeature("hasReferral", true));
    features.add(FeatureFactory.newNumericalFeature("monthlySalary", 500, NumericalFeatureDomain.create(10, 10_000)));
    final TerminationConfig terminationConfig = new TerminationConfig().withScoreCalculationCountLimit(10_000L);
    // for the purpose of this test, only a few steps are necessary
    final SolverConfig solverConfig = SolverConfigBuilder.builder().withTerminationConfig(terminationConfig).build();
    solverConfig.setRandomSeed((long) 23);
    solverConfig.setEnvironmentMode(EnvironmentMode.REPRODUCIBLE);
    final CounterfactualConfig counterfactualConfig = new CounterfactualConfig().withSolverConfig(solverConfig).withGoalThreshold(0.01);
    final CounterfactualExplainer counterfactualExplainer = new CounterfactualExplainer(counterfactualConfig);
    PredictionInput input = new PredictionInput(features);
    PredictionOutput output = new PredictionOutput(goal);
    Prediction prediction = new CounterfactualPrediction(input, output, null, UUID.randomUUID(), 60L);
    final CounterfactualResult counterfactualResult = counterfactualExplainer.explainAsync(prediction, model).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    assertFalse(counterfactualResult.isValid());
}