Examples with PredictionInput org.kie.kogito.explainability.model.PredictionInput used on opensource projects

Example 61 with PredictionInput

use of org.kie.kogito.explainability.model.PredictionInput in project kogito-apps by kiegroup.

the class DatasetEncoderTest method testDatasetEncodingWithNumericData.

@Test
void testDatasetEncodingWithNumericData() {
    List<PredictionInput> perturbedInputs = new LinkedList<>();
    for (int i = 0; i < 10; i++) {
        List<Feature> inputFeatures = new LinkedList<>();
        for (int j = 0; j < 3; j++) {
            inputFeatures.add(TestUtils.getMockedNumericFeature());
        }
        perturbedInputs.add(new PredictionInput(inputFeatures));
    }
    List<Feature> features = new LinkedList<>();
    for (int i = 0; i < 3; i++) {
        features.add(TestUtils.getMockedNumericFeature());
    }
    PredictionInput originalInput = new PredictionInput(features);
    assertEncode(perturbedInputs, originalInput);
}

Example 62 with PredictionInput

use of org.kie.kogito.explainability.model.PredictionInput in project kogito-apps by kiegroup.

the class DatasetEncoderTest method testDatasetEncodingWithTextData.

@Test
void testDatasetEncodingWithTextData() {
    List<PredictionInput> perturbedInputs = new LinkedList<>();
    for (int i = 0; i < 10; i++) {
        List<Feature> inputFeatures = new LinkedList<>();
        for (int j = 0; j < 3; j++) {
            inputFeatures.add(TestUtils.getMockedTextFeature(i + " " + j));
        }
        perturbedInputs.add(new PredictionInput(inputFeatures));
    }
    List<Feature> features = new LinkedList<>();
    for (int i = 0; i < 3; i++) {
        features.add(TestUtils.getMockedTextFeature(i + " " + i));
    }
    PredictionInput originalInput = new PredictionInput(features);
    assertEncode(perturbedInputs, originalInput);
}

Example 63 with PredictionInput

use of org.kie.kogito.explainability.model.PredictionInput in project kogito-apps by kiegroup.

the class DummyModelsLimeExplainerTest method testUnusedFeatureClassification.

@ParameterizedTest
@ValueSource(longs = { 0 })
void testUnusedFeatureClassification(long seed) throws Exception {
    Random random = new Random();
    int idx = 2;
    List<Feature> features = new LinkedList<>();
    features.add(FeatureFactory.newNumericalFeature("f1", 6));
    features.add(FeatureFactory.newNumericalFeature("f2", 3));
    features.add(FeatureFactory.newNumericalFeature("f3", 5));
    PredictionProvider model = TestUtils.getEvenSumModel(idx);
    PredictionInput input = new PredictionInput(features);
    List<PredictionOutput> outputs = model.predictAsync(List.of(input)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    Prediction prediction = new SimplePrediction(input, outputs.get(0));
    LimeConfig limeConfig = new LimeConfig().withSamples(100).withPerturbationContext(new PerturbationContext(seed, random, 1));
    LimeExplainer limeExplainer = new LimeExplainer(limeConfig);
    Map<String, Saliency> saliencyMap = limeExplainer.explainAsync(prediction, model).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    for (Saliency saliency : saliencyMap.values()) {
        assertNotNull(saliency);
        List<FeatureImportance> topFeatures = saliency.getTopFeatures(3);
        assertEquals(3, topFeatures.size());
        assertEquals(1d, ExplainabilityMetrics.impactScore(model, prediction, topFeatures));
    }
    int topK = 1;
    double minimumPositiveStabilityRate = 0.5;
    double minimumNegativeStabilityRate = 0.5;
    TestUtils.assertLimeStability(model, prediction, limeExplainer, topK, minimumPositiveStabilityRate, minimumNegativeStabilityRate);
    List<PredictionInput> inputs = new ArrayList<>();
    for (int i = 0; i < 100; i++) {
        List<Feature> fs = new LinkedList<>();
        fs.add(TestUtils.getMockedNumericFeature());
        fs.add(TestUtils.getMockedNumericFeature());
        fs.add(TestUtils.getMockedNumericFeature());
        inputs.add(new PredictionInput(fs));
    }
    DataDistribution distribution = new PredictionInputsDataDistribution(inputs);
    int k = 2;
    int chunkSize = 10;
    String decision = "sum-even-but" + idx;
    double precision = ExplainabilityMetrics.getLocalSaliencyPrecision(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(precision).isEqualTo(1);
    double recall = ExplainabilityMetrics.getLocalSaliencyRecall(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(recall).isEqualTo(1);
    double f1 = ExplainabilityMetrics.getLocalSaliencyF1(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(f1).isEqualTo(1);
}

Example 64 with PredictionInput

use of org.kie.kogito.explainability.model.PredictionInput in project kogito-apps by kiegroup.

the class DummyModelsLimeExplainerTest method testTextSpamClassification.

@ParameterizedTest
@ValueSource(longs = { 0 })
void testTextSpamClassification(long seed) throws Exception {
    Random random = new Random();
    List<Feature> features = new LinkedList<>();
    Function<String, List<String>> tokenizer = s -> Arrays.asList(s.split(" ").clone());
    features.add(FeatureFactory.newFulltextFeature("f1", "we go here and there", tokenizer));
    features.add(FeatureFactory.newFulltextFeature("f2", "please give me some money", tokenizer));
    features.add(FeatureFactory.newFulltextFeature("f3", "dear friend, please reply", tokenizer));
    PredictionInput input = new PredictionInput(features);
    PredictionProvider model = TestUtils.getDummyTextClassifier();
    List<PredictionOutput> outputs = model.predictAsync(List.of(input)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    Prediction prediction = new SimplePrediction(input, outputs.get(0));
    LimeConfig limeConfig = new LimeConfig().withSamples(100).withPerturbationContext(new PerturbationContext(seed, random, 1));
    LimeExplainer limeExplainer = new LimeExplainer(limeConfig);
    Map<String, Saliency> saliencyMap = limeExplainer.explainAsync(prediction, model).toCompletableFuture().get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    for (Saliency saliency : saliencyMap.values()) {
        assertNotNull(saliency);
        List<FeatureImportance> topFeatures = saliency.getPositiveFeatures(1);
        assertEquals(1, topFeatures.size());
        assertEquals(1d, ExplainabilityMetrics.impactScore(model, prediction, topFeatures));
    }
    int topK = 1;
    double minimumPositiveStabilityRate = 0.5;
    double minimumNegativeStabilityRate = 0.2;
    TestUtils.assertLimeStability(model, prediction, limeExplainer, topK, minimumPositiveStabilityRate, minimumNegativeStabilityRate);
    List<PredictionInput> inputs = new ArrayList<>();
    for (int i = 0; i < 100; i++) {
        List<Feature> fs = new LinkedList<>();
        fs.add(TestUtils.getMockedNumericFeature());
        fs.add(TestUtils.getMockedNumericFeature());
        fs.add(TestUtils.getMockedNumericFeature());
        inputs.add(new PredictionInput(fs));
    }
    DataDistribution distribution = new PredictionInputsDataDistribution(inputs);
    int k = 2;
    int chunkSize = 10;
    String decision = "spam";
    double precision = ExplainabilityMetrics.getLocalSaliencyPrecision(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(precision).isEqualTo(1);
    double recall = ExplainabilityMetrics.getLocalSaliencyRecall(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(recall).isEqualTo(1);
    double f1 = ExplainabilityMetrics.getLocalSaliencyF1(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(f1).isEqualTo(1);
}

Example 65 with PredictionInput

use of org.kie.kogito.explainability.model.PredictionInput in project kogito-apps by kiegroup.

the class FairnessMetricsTest method testGroupDIRTextClassifier.

@Test
void testGroupDIRTextClassifier() throws ExecutionException, InterruptedException {
    List<PredictionInput> testInputs = getTestInputs();
    PredictionProvider model = TestUtils.getDummyTextClassifier();
    Predicate<PredictionInput> selector = predictionInput -> DataUtils.textify(predictionInput).contains("please");
    Output output = new Output("spam", Type.BOOLEAN, new Value(false), 1.0);
    double dir = FairnessMetrics.groupDisparateImpactRatio(selector, testInputs, model, output);
    assertThat(dir).isPositive();
}