Examples with SimplePrediction org.kie.kogito.explainability.model.SimplePrediction used on opensource projects

Example 71 with SimplePrediction

use of org.kie.kogito.explainability.model.SimplePrediction in project kogito-apps by kiegroup.

the class FairnessMetricsTest method getTestData.

private List<Prediction> getTestData() {
    List<Prediction> data = new ArrayList<>();
    Function<String, List<String>> tokenizer = s -> Arrays.asList(s.split(" ").clone());
    List<Feature> features = new ArrayList<>();
    features.add(FeatureFactory.newFulltextFeature("subject", "urgent inquiry", tokenizer));
    features.add(FeatureFactory.newFulltextFeature("text", "please give me some money", tokenizer));
    Output output = new Output("spam", Type.BOOLEAN, new Value(true), 1);
    data.add(new SimplePrediction(new PredictionInput(features), new PredictionOutput(List.of(output))));
    features = new ArrayList<>();
    features.add(FeatureFactory.newFulltextFeature("subject", "do not reply", tokenizer));
    features.add(FeatureFactory.newFulltextFeature("text", "if you asked to reset your password, ignore this", tokenizer));
    output = new Output("spam", Type.BOOLEAN, new Value(false), 1);
    data.add(new SimplePrediction(new PredictionInput(features), new PredictionOutput(List.of(output))));
    features = new ArrayList<>();
    features.add(FeatureFactory.newFulltextFeature("subject", "please reply", tokenizer));
    features.add(FeatureFactory.newFulltextFeature("text", "we got money matter! please reply", tokenizer));
    output = new Output("spam", Type.BOOLEAN, new Value(true), 1);
    data.add(new SimplePrediction(new PredictionInput(features), new PredictionOutput(List.of(output))));
    features = new ArrayList<>();
    features.add(FeatureFactory.newFulltextFeature("subject", "inquiry", tokenizer));
    features.add(FeatureFactory.newFulltextFeature("text", "would you like to get a 100% secure way to invest your money?", tokenizer));
    output = new Output("spam", Type.BOOLEAN, new Value(true), 1);
    data.add(new SimplePrediction(new PredictionInput(features), new PredictionOutput(List.of(output))));
    features = new ArrayList<>();
    features.add(FeatureFactory.newFulltextFeature("subject", "clear some space", tokenizer));
    features.add(FeatureFactory.newFulltextFeature("text", "you just finished your space, upgrade today for 1 $ a week", tokenizer));
    output = new Output("spam", Type.BOOLEAN, new Value(false), 1);
    data.add(new SimplePrediction(new PredictionInput(features), new PredictionOutput(List.of(output))));
    features = new ArrayList<>();
    features.add(FeatureFactory.newFulltextFeature("subject", "prize waiting", tokenizer));
    features.add(FeatureFactory.newFulltextFeature("text", "you are the lucky winner of a 100k $ prize", tokenizer));
    output = new Output("spam", Type.BOOLEAN, new Value(true), 1);
    data.add(new SimplePrediction(new PredictionInput(features), new PredictionOutput(List.of(output))));
    features = new ArrayList<>();
    features.add(FeatureFactory.newFulltextFeature("subject", "urgent matter", tokenizer));
    features.add(FeatureFactory.newFulltextFeature("text", "we got an urgent inquiry for you to answer.", tokenizer));
    output = new Output("spam", Type.BOOLEAN, new Value(true), 1);
    data.add(new SimplePrediction(new PredictionInput(features), new PredictionOutput(List.of(output))));
    features = new ArrayList<>();
    features.add(FeatureFactory.newFulltextFeature("subject", "password change", tokenizer));
    features.add(FeatureFactory.newFulltextFeature("text", "you just requested to change your password", tokenizer));
    output = new Output("spam", Type.BOOLEAN, new Value(false), 1);
    data.add(new SimplePrediction(new PredictionInput(features), new PredictionOutput(List.of(output))));
    features = new ArrayList<>();
    features.add(FeatureFactory.newFulltextFeature("subject", "password stolen", tokenizer));
    features.add(FeatureFactory.newFulltextFeature("text", "we stole your password, if you want it back, send some money .", tokenizer));
    output = new Output("spam", Type.BOOLEAN, new Value(true), 1);
    data.add(new SimplePrediction(new PredictionInput(features), new PredictionOutput(List.of(output))));
    return data;
}

Example 72 with SimplePrediction

use of org.kie.kogito.explainability.model.SimplePrediction in project kogito-apps by kiegroup.

the class ValidationUtilsTest method testStableEval.

@Test
void testStableEval() throws ExecutionException, InterruptedException, TimeoutException, ValidationUtils.ValidationException {
    for (int n = 0; n < 10; n++) {
        Random random = new Random();
        PerturbationContext perturbationContext = new PerturbationContext(4L, random, 1);
        LimeConfig config = new LimeConfig().withPerturbationContext(perturbationContext);
        LimeExplainer explainer = new LimeExplainer(config);
        PredictionProvider model = TestUtils.getSumThresholdModel(0.1, 0.1);
        List<Feature> features = new ArrayList<>();
        for (int i = 0; i < 4; i++) {
            features.add(FeatureFactory.newNumericalFeature("f-" + i, Type.NUMBER.randomValue(perturbationContext).asNumber()));
        }
        PredictionInput input = new PredictionInput(features);
        List<PredictionOutput> outputs = model.predictAsync(List.of(input)).get(Config.DEFAULT_ASYNC_TIMEOUT, Config.DEFAULT_ASYNC_TIMEUNIT);
        Prediction prediction = new SimplePrediction(input, outputs.get(0));
        int topK = 1;
        double posScore = 0.6;
        double minScore = 0.6;
        ValidationUtils.validateLocalSaliencyStability(model, prediction, explainer, topK, posScore, minScore);
    }
}

Example 73 with SimplePrediction

use of org.kie.kogito.explainability.model.SimplePrediction in project kogito-apps by kiegroup.

the class FraudScoringDmnLimeExplainerTest method testExplanationWithDataDistribution.

@Test
void testExplanationWithDataDistribution() throws ExecutionException, InterruptedException, TimeoutException {
    PredictionProvider model = getModel();
    List<PredictionInput> samples = DmnTestUtils.randomFraudScoringInputs();
    List<PredictionInput> inputs = samples.subList(0, 10);
    Random random = new Random();
    random.setSeed(0);
    PerturbationContext perturbationContext = new PerturbationContext(random, 1);
    LimeConfig initialConfig = new LimeConfig().withSamples(10).withDataDistribution(new PredictionInputsDataDistribution(inputs)).withPerturbationContext(perturbationContext);
    LimeExplainer limeExplainer = new LimeExplainer(initialConfig);
    PredictionInput testPredictionInput = getTestInput();
    List<PredictionOutput> testPredictionOutputs = model.predictAsync(List.of(testPredictionInput)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    Prediction instance = new SimplePrediction(testPredictionInput, testPredictionOutputs.get(0));
    assertDoesNotThrow(() -> ValidationUtils.validateLocalSaliencyStability(model, instance, limeExplainer, 1, 0.5, 0.5));
}

Example 74 with SimplePrediction

use of org.kie.kogito.explainability.model.SimplePrediction in project kogito-apps by kiegroup.

the class LoanEligibilityDmnLimeExplainerTest method testExplanationStabilityWithOptimization.

@Test
void testExplanationStabilityWithOptimization() throws ExecutionException, InterruptedException, TimeoutException {
    PredictionProvider model = getModel();
    List<PredictionInput> samples = DmnTestUtils.randomLoanEligibilityInputs();
    List<PredictionOutput> predictionOutputs = model.predictAsync(samples.subList(0, 5)).get();
    List<Prediction> predictions = DataUtils.getPredictions(samples, predictionOutputs);
    long seed = 0;
    LimeConfigOptimizer limeConfigOptimizer = new LimeConfigOptimizer().withDeterministicExecution(true).withStepCountLimit(20);
    Random random = new Random();
    PerturbationContext perturbationContext = new PerturbationContext(seed, random, 1);
    LimeConfig initialConfig = new LimeConfig().withPerturbationContext(perturbationContext);
    LimeConfig optimizedConfig = limeConfigOptimizer.optimize(initialConfig, predictions, model);
    assertThat(optimizedConfig).isNotSameAs(initialConfig);
    LimeExplainer limeExplainer = new LimeExplainer(optimizedConfig);
    PredictionInput testPredictionInput = getTestInput();
    List<PredictionOutput> testPredictionOutputs = model.predictAsync(List.of(testPredictionInput)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    Prediction instance = new SimplePrediction(testPredictionInput, testPredictionOutputs.get(0));
    assertDoesNotThrow(() -> ValidationUtils.validateLocalSaliencyStability(model, instance, limeExplainer, 1, 0.5, 0.5));
}

Example 75 with SimplePrediction

use of org.kie.kogito.explainability.model.SimplePrediction in project kogito-apps by kiegroup.

the class PrequalificationDmnLimeExplainerTest method testPrequalificationDMNExplanation.

@Test
void testPrequalificationDMNExplanation() throws ExecutionException, InterruptedException, TimeoutException {
    PredictionProvider model = getModel();
    PredictionInput predictionInput = getTestInput();
    Random random = new Random();
    PerturbationContext perturbationContext = new PerturbationContext(0L, random, 1);
    LimeConfig limeConfig = new LimeConfig().withSamples(10).withPerturbationContext(perturbationContext);
    LimeExplainer limeExplainer = new LimeExplainer(limeConfig);
    List<PredictionOutput> predictionOutputs = model.predictAsync(List.of(predictionInput)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    Prediction prediction = new SimplePrediction(predictionInput, predictionOutputs.get(0));
    Map<String, Saliency> saliencyMap = limeExplainer.explainAsync(prediction, model).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    for (Saliency saliency : saliencyMap.values()) {
        assertNotNull(saliency);
        List<FeatureImportance> topFeatures = saliency.getTopFeatures(2);
        if (!topFeatures.isEmpty()) {
            assertThat(ExplainabilityMetrics.impactScore(model, prediction, topFeatures)).isPositive();
        }
    }
    assertDoesNotThrow(() -> ValidationUtils.validateLocalSaliencyStability(model, prediction, limeExplainer, 1, 0.3, 0.3));
    String decision = "LLPA";
    List<PredictionInput> inputs = new ArrayList<>();
    for (int n = 0; n < 10; n++) {
        inputs.add(new PredictionInput(DataUtils.perturbFeatures(predictionInput.getFeatures(), perturbationContext)));
    }
    DataDistribution distribution = new PredictionInputsDataDistribution(inputs);
    int k = 2;
    int chunkSize = 2;
    double f1 = ExplainabilityMetrics.getLocalSaliencyF1(decision, model, limeExplainer, distribution, k, chunkSize);
    AssertionsForClassTypes.assertThat(f1).isBetween(0.5d, 1d);
}