Examples with PredictionOutput org.kie.kogito.explainability.model.PredictionOutput used on opensource projects

Example 46 with PredictionOutput

use of org.kie.kogito.explainability.model.PredictionOutput in project kogito-apps by kiegroup.

the class OpenNLPLimeExplainerTest method getModel.

private PredictionProvider getModel() throws IOException {
    InputStream is = getClass().getResourceAsStream("/opennlp/langdetect-183.bin");
    LanguageDetectorModel languageDetectorModel = new LanguageDetectorModel(is);
    LanguageDetector languageDetector = new LanguageDetectorME(languageDetectorModel);
    return inputs -> CompletableFuture.supplyAsync(() -> {
        List<PredictionOutput> results = new LinkedList<>();
        for (PredictionInput predictionInput : inputs) {
            StringBuilder builder = new StringBuilder();
            for (Feature f : predictionInput.getFeatures()) {
                if (builder.length() > 0) {
                    builder.append(' ');
                }
                builder.append(f.getValue().asString());
            }
            Language language = languageDetector.predictLanguage(builder.toString());
            PredictionOutput predictionOutput = new PredictionOutput(List.of(new Output("lang", Type.TEXT, new Value(language.getLang()), language.getConfidence())));
            results.add(predictionOutput);
        }
        return results;
    });
}

Example 47 with PredictionOutput

use of org.kie.kogito.explainability.model.PredictionOutput in project kogito-apps by kiegroup.

the class OpenNLPLimeExplainerTest method testOpenNLPLangDetect.

@ParameterizedTest
@ValueSource(longs = { 0 })
void testOpenNLPLangDetect(long seed) throws Exception {
    Random random = new Random();
    LimeConfig limeConfig = new LimeConfig().withSamples(10).withPerturbationContext(new PerturbationContext(seed, random, 1));
    LimeExplainer limeExplainer = new LimeExplainer(limeConfig);
    PredictionProvider model = getModel();
    Function<String, List<String>> tokenizer = getTokenizer();
    PredictionInput testInput = getTestInput(tokenizer);
    List<PredictionOutput> predictionOutputs = model.predictAsync(List.of(testInput)).get();
    assertNotNull(predictionOutputs);
    assertFalse(predictionOutputs.isEmpty());
    PredictionOutput output = predictionOutputs.get(0);
    assertNotNull(output);
    assertNotNull(output.getOutputs());
    assertEquals(1, output.getOutputs().size());
    assertEquals("ita", output.getOutputs().get(0).getValue().asString());
    assertEquals(0.03, output.getOutputs().get(0).getScore(), 1e-2);
    Prediction prediction = new SimplePrediction(testInput, output);
    Map<String, Saliency> saliencyMap = limeExplainer.explainAsync(prediction, model).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    for (Saliency saliency : saliencyMap.values()) {
        assertNotNull(saliency);
        double i1 = ExplainabilityMetrics.impactScore(model, prediction, saliency.getPositiveFeatures(3));
        assertEquals(1d, i1);
    }
    assertDoesNotThrow(() -> ValidationUtils.validateLocalSaliencyStability(model, prediction, limeExplainer, 2, 0.6, 0.6));
    List<PredictionInput> inputs = getSamples(tokenizer);
    String decision = "lang";
    DataDistribution distribution = new PredictionInputsDataDistribution(inputs);
    int k = 2;
    int chunkSize = 2;
    double f1 = ExplainabilityMetrics.getLocalSaliencyF1(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(f1).isBetween(0.5d, 1d);
}

Example 48 with PredictionOutput

use of org.kie.kogito.explainability.model.PredictionOutput in project kogito-apps by kiegroup.

the class OpenNLPLimeExplainerTest method testExplanationWeightedStabilityWithOptimization.

@Test
void testExplanationWeightedStabilityWithOptimization() throws ExecutionException, InterruptedException, TimeoutException, IOException {
    PredictionProvider model = getModel();
    List<PredictionInput> samples = getSamples(getTokenizer());
    List<PredictionOutput> predictionOutputs = model.predictAsync(samples.subList(0, 5)).get();
    List<Prediction> predictions = DataUtils.getPredictions(samples, predictionOutputs);
    long seed = 0;
    LimeConfigOptimizer limeConfigOptimizer = new LimeConfigOptimizer().withDeterministicExecution(true).withSampling(false).withStepCountLimit(30).withWeightedStability(0.4, 0.6);
    Random random = new Random();
    LimeConfig limeConfig = new LimeConfig().withSamples(10).withPerturbationContext(new PerturbationContext(seed, random, 1));
    LimeConfig optimizedConfig = limeConfigOptimizer.optimize(limeConfig, predictions, model);
    Assertions.assertThat(optimizedConfig).isNotSameAs(limeConfig);
    LimeExplainer limeExplainer = new LimeExplainer(optimizedConfig);
    PredictionInput testPredictionInput = getTestInput(getTokenizer());
    List<PredictionOutput> testPredictionOutputs = model.predictAsync(List.of(testPredictionInput)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    Prediction instance = new SimplePrediction(testPredictionInput, testPredictionOutputs.get(0));
    assertDoesNotThrow(() -> ValidationUtils.validateLocalSaliencyStability(model, instance, limeExplainer, 1, 0.9, 0.8));
}

Example 49 with PredictionOutput

use of org.kie.kogito.explainability.model.PredictionOutput in project kogito-apps by kiegroup.

the class OpenNLPLimeExplainerTest method testExplanationStabilityWithOptimization.

@Test
void testExplanationStabilityWithOptimization() throws ExecutionException, InterruptedException, TimeoutException, IOException {
    PredictionProvider model = getModel();
    List<PredictionInput> samples = getSamples(getTokenizer());
    List<PredictionOutput> predictionOutputs = model.predictAsync(samples.subList(0, 5)).get();
    List<Prediction> predictions = DataUtils.getPredictions(samples, predictionOutputs);
    long seed = 0;
    LimeConfigOptimizer limeConfigOptimizer = new LimeConfigOptimizer().withDeterministicExecution(true).withSampling(false).withStepCountLimit(30);
    Random random = new Random();
    LimeConfig limeConfig = new LimeConfig().withSamples(10).withPerturbationContext(new PerturbationContext(seed, random, 1));
    LimeConfig optimizedConfig = limeConfigOptimizer.optimize(limeConfig, predictions, model);
    Assertions.assertThat(optimizedConfig).isNotSameAs(limeConfig);
    LimeExplainer limeExplainer = new LimeExplainer(optimizedConfig);
    PredictionInput testPredictionInput = getTestInput(getTokenizer());
    List<PredictionOutput> testPredictionOutputs = model.predictAsync(List.of(testPredictionInput)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    Prediction instance = new SimplePrediction(testPredictionInput, testPredictionOutputs.get(0));
    assertDoesNotThrow(() -> ValidationUtils.validateLocalSaliencyStability(model, instance, limeExplainer, 1, 0.8, 0.8));
}

Example 50 with PredictionOutput

use of org.kie.kogito.explainability.model.PredictionOutput in project kogito-apps by kiegroup.

the class OpenNLPLimeExplainerTest method testExplanationImpactScoreWithOptimization.

@Test
void testExplanationImpactScoreWithOptimization() throws ExecutionException, InterruptedException, IOException {
    PredictionProvider model = getModel();
    List<PredictionInput> samples = getSamples(getTokenizer());
    List<PredictionOutput> predictionOutputs = model.predictAsync(samples.subList(0, 5)).get();
    List<Prediction> predictions = DataUtils.getPredictions(samples, predictionOutputs);
    long seed = 0;
    LimeConfigOptimizer limeConfigOptimizer = new LimeConfigOptimizer().withDeterministicExecution(true).forImpactScore().withSampling(false).withStepCountLimit(30);
    Random random = new Random();
    LimeConfig limeConfig = new LimeConfig().withSamples(10).withPerturbationContext(new PerturbationContext(seed, random, 1));
    LimeConfig optimizedConfig = limeConfigOptimizer.optimize(limeConfig, predictions, model);
    Assertions.assertThat(optimizedConfig).isNotSameAs(limeConfig);
}