Search in sources :

Example 36 with PredictionProvider

use of org.kie.kogito.explainability.model.PredictionProvider in project kogito-apps by kiegroup.

the class OpenNLPLimeExplainerTest method testExplanationImpactScoreWithOptimization.

@Test
void testExplanationImpactScoreWithOptimization() throws ExecutionException, InterruptedException, IOException {
    PredictionProvider model = getModel();
    List<PredictionInput> samples = getSamples(getTokenizer());
    List<PredictionOutput> predictionOutputs = model.predictAsync(samples.subList(0, 5)).get();
    List<Prediction> predictions = DataUtils.getPredictions(samples, predictionOutputs);
    long seed = 0;
    LimeConfigOptimizer limeConfigOptimizer = new LimeConfigOptimizer().withDeterministicExecution(true).forImpactScore().withSampling(false).withStepCountLimit(30);
    Random random = new Random();
    LimeConfig limeConfig = new LimeConfig().withSamples(10).withPerturbationContext(new PerturbationContext(seed, random, 1));
    LimeConfig optimizedConfig = limeConfigOptimizer.optimize(limeConfig, predictions, model);
    Assertions.assertThat(optimizedConfig).isNotSameAs(limeConfig);
}
Also used : PerturbationContext(org.kie.kogito.explainability.model.PerturbationContext) Random(java.util.Random) PredictionInput(org.kie.kogito.explainability.model.PredictionInput) PredictionOutput(org.kie.kogito.explainability.model.PredictionOutput) Prediction(org.kie.kogito.explainability.model.Prediction) SimplePrediction(org.kie.kogito.explainability.model.SimplePrediction) PredictionProvider(org.kie.kogito.explainability.model.PredictionProvider) LimeConfigOptimizer(org.kie.kogito.explainability.local.lime.optim.LimeConfigOptimizer) LimeConfig(org.kie.kogito.explainability.local.lime.LimeConfig) Test(org.junit.jupiter.api.Test) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 37 with PredictionProvider

use of org.kie.kogito.explainability.model.PredictionProvider in project kogito-apps by kiegroup.

the class OpenNLPPDPExplainerTest method testOpenNLPLangDetect.

@Test
void testOpenNLPLangDetect() throws Exception {
    PartialDependencePlotExplainer partialDependencePlotExplainer = new PartialDependencePlotExplainer();
    InputStream is = getClass().getResourceAsStream("/opennlp/langdetect-183.bin");
    LanguageDetectorModel languageDetectorModel = new LanguageDetectorModel(is);
    LanguageDetector languageDetector = new LanguageDetectorME(languageDetectorModel);
    PredictionProvider model = inputs -> CompletableFuture.supplyAsync(() -> {
        List<PredictionOutput> results = new ArrayList<>();
        for (PredictionInput predictionInput : inputs) {
            StringBuilder builder = new StringBuilder();
            for (Feature f : predictionInput.getFeatures()) {
                if (builder.length() > 0) {
                    builder.append(' ');
                }
                builder.append(f.getValue().asString());
            }
            Language language = languageDetector.predictLanguage(builder.toString());
            PredictionOutput predictionOutput = new PredictionOutput(List.of(new Output("lang", Type.TEXT, new Value(language.getLang()), language.getConfidence())));
            results.add(predictionOutput);
        }
        return results;
    });
    List<String> texts = List.of("we want your money", "please reply quickly", "you are the lucky winner", "italiani, spaghetti pizza mandolino", "guten tag", "allez les bleus", "daje roma");
    List<Prediction> predictions = new ArrayList<>();
    for (String text : texts) {
        List<Feature> features = new ArrayList<>();
        features.add(FeatureFactory.newFulltextFeature("text", text));
        PredictionInput predictionInput = new PredictionInput(features);
        PredictionOutput predictionOutput = model.predictAsync(List.of(predictionInput)).get().get(0);
        predictions.add(new SimplePrediction(predictionInput, predictionOutput));
    }
    List<PartialDependenceGraph> pdps = partialDependencePlotExplainer.explainFromPredictions(model, predictions);
    assertThat(pdps).isNotEmpty();
}
Also used : LanguageDetectorME(opennlp.tools.langdetect.LanguageDetectorME) FeatureFactory(org.kie.kogito.explainability.model.FeatureFactory) SimplePrediction(org.kie.kogito.explainability.model.SimplePrediction) LanguageDetectorModel(opennlp.tools.langdetect.LanguageDetectorModel) Feature(org.kie.kogito.explainability.model.Feature) Prediction(org.kie.kogito.explainability.model.Prediction) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) CompletableFuture(java.util.concurrent.CompletableFuture) Value(org.kie.kogito.explainability.model.Value) Type(org.kie.kogito.explainability.model.Type) PredictionProvider(org.kie.kogito.explainability.model.PredictionProvider) ArrayList(java.util.ArrayList) Test(org.junit.jupiter.api.Test) PredictionInput(org.kie.kogito.explainability.model.PredictionInput) List(java.util.List) PartialDependencePlotExplainer(org.kie.kogito.explainability.global.pdp.PartialDependencePlotExplainer) Language(opennlp.tools.langdetect.Language) LanguageDetector(opennlp.tools.langdetect.LanguageDetector) PartialDependenceGraph(org.kie.kogito.explainability.model.PartialDependenceGraph) Output(org.kie.kogito.explainability.model.Output) PredictionOutput(org.kie.kogito.explainability.model.PredictionOutput) InputStream(java.io.InputStream) SimplePrediction(org.kie.kogito.explainability.model.SimplePrediction) PredictionInput(org.kie.kogito.explainability.model.PredictionInput) InputStream(java.io.InputStream) SimplePrediction(org.kie.kogito.explainability.model.SimplePrediction) Prediction(org.kie.kogito.explainability.model.Prediction) LanguageDetectorME(opennlp.tools.langdetect.LanguageDetectorME) ArrayList(java.util.ArrayList) PredictionProvider(org.kie.kogito.explainability.model.PredictionProvider) Feature(org.kie.kogito.explainability.model.Feature) LanguageDetector(opennlp.tools.langdetect.LanguageDetector) PartialDependencePlotExplainer(org.kie.kogito.explainability.global.pdp.PartialDependencePlotExplainer) Language(opennlp.tools.langdetect.Language) LanguageDetectorModel(opennlp.tools.langdetect.LanguageDetectorModel) PredictionOutput(org.kie.kogito.explainability.model.PredictionOutput) Output(org.kie.kogito.explainability.model.Output) PredictionOutput(org.kie.kogito.explainability.model.PredictionOutput) Value(org.kie.kogito.explainability.model.Value) PartialDependenceGraph(org.kie.kogito.explainability.model.PartialDependenceGraph) Test(org.junit.jupiter.api.Test)

Example 38 with PredictionProvider

use of org.kie.kogito.explainability.model.PredictionProvider in project kogito-apps by kiegroup.

the class PmmlCompoundScorecardLimeExplainerTest method testPMMLCompoundScorecard.

@Test
void testPMMLCompoundScorecard() throws Exception {
    Random random = new Random();
    LimeConfig limeConfig = new LimeConfig().withSamples(10).withPerturbationContext(new PerturbationContext(0L, random, 1));
    LimeExplainer limeExplainer = new LimeExplainer(limeConfig);
    PredictionInput input = getTestInput();
    PredictionProvider model = getModel();
    List<PredictionOutput> predictionOutputs = model.predictAsync(List.of(input)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    assertThat(predictionOutputs).isNotNull();
    assertThat(predictionOutputs).isNotEmpty();
    PredictionOutput output = predictionOutputs.get(0);
    assertThat(output).isNotNull();
    Prediction prediction = new SimplePrediction(input, output);
    Map<String, Saliency> saliencyMap = limeExplainer.explainAsync(prediction, model).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    for (Saliency saliency : saliencyMap.values()) {
        assertThat(saliency).isNotNull();
        double v = ExplainabilityMetrics.impactScore(model, prediction, saliency.getTopFeatures(2));
        assertThat(v).isEqualTo(1d);
    }
    assertDoesNotThrow(() -> ValidationUtils.validateLocalSaliencyStability(model, prediction, limeExplainer, 1, 0.5, 0.5));
    List<PredictionInput> inputs = getSamples();
    DataDistribution distribution = new PredictionInputsDataDistribution(inputs);
    String decision = "score";
    int k = 1;
    int chunkSize = 2;
    double f1 = ExplainabilityMetrics.getLocalSaliencyF1(decision, model, limeExplainer, distribution, k, chunkSize);
    AssertionsForClassTypes.assertThat(f1).isBetween(0d, 1d);
}
Also used : SimplePrediction(org.kie.kogito.explainability.model.SimplePrediction) PerturbationContext(org.kie.kogito.explainability.model.PerturbationContext) PredictionInput(org.kie.kogito.explainability.model.PredictionInput) LimeExplainer(org.kie.kogito.explainability.local.lime.LimeExplainer) Prediction(org.kie.kogito.explainability.model.Prediction) SimplePrediction(org.kie.kogito.explainability.model.SimplePrediction) Saliency(org.kie.kogito.explainability.model.Saliency) PredictionProvider(org.kie.kogito.explainability.model.PredictionProvider) LimeConfig(org.kie.kogito.explainability.local.lime.LimeConfig) Random(java.util.Random) PredictionInputsDataDistribution(org.kie.kogito.explainability.model.PredictionInputsDataDistribution) DataDistribution(org.kie.kogito.explainability.model.DataDistribution) PredictionOutput(org.kie.kogito.explainability.model.PredictionOutput) PredictionInputsDataDistribution(org.kie.kogito.explainability.model.PredictionInputsDataDistribution) Test(org.junit.jupiter.api.Test)

Example 39 with PredictionProvider

use of org.kie.kogito.explainability.model.PredictionProvider in project kogito-apps by kiegroup.

the class PmmlCompoundScorecardLimeExplainerTest method testExplanationWeightedStabilityWithOptimization.

@Test
void testExplanationWeightedStabilityWithOptimization() throws ExecutionException, InterruptedException, TimeoutException {
    PredictionProvider model = getModel();
    List<PredictionInput> samples = getSamples();
    List<PredictionOutput> predictionOutputs = model.predictAsync(samples.subList(0, 5)).get();
    List<Prediction> predictions = DataUtils.getPredictions(samples, predictionOutputs);
    long seed = 0;
    LimeConfigOptimizer limeConfigOptimizer = new LimeConfigOptimizer().withDeterministicExecution(true).withWeightedStability(0.4, 0.6);
    Random random = new Random();
    PerturbationContext perturbationContext = new PerturbationContext(seed, random, 1);
    LimeConfig initialConfig = new LimeConfig().withSamples(10).withPerturbationContext(perturbationContext);
    LimeConfig optimizedConfig = limeConfigOptimizer.optimize(initialConfig, predictions, model);
    assertThat(optimizedConfig).isNotSameAs(initialConfig);
    LimeExplainer limeExplainer = new LimeExplainer(optimizedConfig);
    PredictionInput testPredictionInput = getTestInput();
    List<PredictionOutput> testPredictionOutputs = model.predictAsync(List.of(testPredictionInput)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    Prediction instance = new SimplePrediction(testPredictionInput, testPredictionOutputs.get(0));
    assertDoesNotThrow(() -> ValidationUtils.validateLocalSaliencyStability(model, instance, limeExplainer, 1, 0.5, 0.7));
}
Also used : SimplePrediction(org.kie.kogito.explainability.model.SimplePrediction) PerturbationContext(org.kie.kogito.explainability.model.PerturbationContext) PredictionInput(org.kie.kogito.explainability.model.PredictionInput) LimeExplainer(org.kie.kogito.explainability.local.lime.LimeExplainer) Prediction(org.kie.kogito.explainability.model.Prediction) SimplePrediction(org.kie.kogito.explainability.model.SimplePrediction) PredictionProvider(org.kie.kogito.explainability.model.PredictionProvider) LimeConfig(org.kie.kogito.explainability.local.lime.LimeConfig) Random(java.util.Random) PredictionOutput(org.kie.kogito.explainability.model.PredictionOutput) LimeConfigOptimizer(org.kie.kogito.explainability.local.lime.optim.LimeConfigOptimizer) Test(org.junit.jupiter.api.Test)

Example 40 with PredictionProvider

use of org.kie.kogito.explainability.model.PredictionProvider in project kogito-apps by kiegroup.

the class PmmlCompoundScorecardLimeExplainerTest method getModel.

private PredictionProvider getModel() {
    return inputs -> CompletableFuture.supplyAsync(() -> {
        List<PredictionOutput> outputs = new ArrayList<>(inputs.size());
        for (PredictionInput predictionInput : inputs) {
            List<Feature> inputFeatures = predictionInput.getFeatures();
            CompoundNestedPredicateScorecardExecutor pmmlModel = new CompoundNestedPredicateScorecardExecutor(inputFeatures.get(0).getValue().asNumber(), inputFeatures.get(1).getValue().asString());
            PMML4Result result = pmmlModel.execute(compoundScoreCardRuntime);
            Map<String, Object> resultVariables = result.getResultVariables();
            String score = "" + resultVariables.get(CompoundNestedPredicateScorecardExecutor.TARGET_FIELD);
            String reason1 = "" + resultVariables.get(CompoundNestedPredicateScorecardExecutor.REASON_CODE1_FIELD);
            PredictionOutput predictionOutput = new PredictionOutput(List.of(new Output("score", Type.TEXT, new Value(score), 1d), new Output("reason1", Type.TEXT, new Value(reason1), 1d)));
            outputs.add(predictionOutput);
        }
        return outputs;
    });
}
Also used : FeatureFactory(org.kie.kogito.explainability.model.FeatureFactory) PredictionInputsDataDistribution(org.kie.kogito.explainability.model.PredictionInputsDataDistribution) PerturbationContext(org.kie.kogito.explainability.model.PerturbationContext) PMMLRuntime(org.kie.pmml.api.runtime.PMMLRuntime) Feature(org.kie.kogito.explainability.model.Feature) Prediction(org.kie.kogito.explainability.model.Prediction) URISyntaxException(java.net.URISyntaxException) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) AssertionsForClassTypes(org.assertj.core.api.AssertionsForClassTypes) TimeoutException(java.util.concurrent.TimeoutException) Random(java.util.Random) CompletableFuture(java.util.concurrent.CompletableFuture) PMML4Result(org.kie.api.pmml.PMML4Result) Value(org.kie.kogito.explainability.model.Value) DataDistribution(org.kie.kogito.explainability.model.DataDistribution) Saliency(org.kie.kogito.explainability.model.Saliency) PMMLRuntimeFactoryInternal.getPMMLRuntime(org.kie.pmml.evaluator.assembler.factories.PMMLRuntimeFactoryInternal.getPMMLRuntime) ArrayList(java.util.ArrayList) BeforeAll(org.junit.jupiter.api.BeforeAll) Map(java.util.Map) LimeConfig(org.kie.kogito.explainability.local.lime.LimeConfig) PredictionOutput(org.kie.kogito.explainability.model.PredictionOutput) LimeConfigOptimizer(org.kie.kogito.explainability.local.lime.optim.LimeConfigOptimizer) SimplePrediction(org.kie.kogito.explainability.model.SimplePrediction) LimeExplainer(org.kie.kogito.explainability.local.lime.LimeExplainer) DataUtils(org.kie.kogito.explainability.utils.DataUtils) Type(org.kie.kogito.explainability.model.Type) PredictionProvider(org.kie.kogito.explainability.model.PredictionProvider) ExecutionException(java.util.concurrent.ExecutionException) Test(org.junit.jupiter.api.Test) PredictionInput(org.kie.kogito.explainability.model.PredictionInput) List(java.util.List) ExplainabilityMetrics(org.kie.kogito.explainability.utils.ExplainabilityMetrics) Output(org.kie.kogito.explainability.model.Output) ValidationUtils(org.kie.kogito.explainability.utils.ValidationUtils) Config(org.kie.kogito.explainability.Config) Assertions.assertDoesNotThrow(org.junit.jupiter.api.Assertions.assertDoesNotThrow) PMML4Result(org.kie.api.pmml.PMML4Result) PredictionInput(org.kie.kogito.explainability.model.PredictionInput) ArrayList(java.util.ArrayList) Feature(org.kie.kogito.explainability.model.Feature) PredictionOutput(org.kie.kogito.explainability.model.PredictionOutput) PredictionOutput(org.kie.kogito.explainability.model.PredictionOutput) Output(org.kie.kogito.explainability.model.Output) Value(org.kie.kogito.explainability.model.Value)

Aggregations

PredictionProvider (org.kie.kogito.explainability.model.PredictionProvider)158 Prediction (org.kie.kogito.explainability.model.Prediction)134 PredictionInput (org.kie.kogito.explainability.model.PredictionInput)134 PredictionOutput (org.kie.kogito.explainability.model.PredictionOutput)126 Test (org.junit.jupiter.api.Test)109 SimplePrediction (org.kie.kogito.explainability.model.SimplePrediction)99 Random (java.util.Random)91 Feature (org.kie.kogito.explainability.model.Feature)76 ArrayList (java.util.ArrayList)73 PerturbationContext (org.kie.kogito.explainability.model.PerturbationContext)69 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)64 LimeConfig (org.kie.kogito.explainability.local.lime.LimeConfig)59 LimeExplainer (org.kie.kogito.explainability.local.lime.LimeExplainer)54 Output (org.kie.kogito.explainability.model.Output)45 Saliency (org.kie.kogito.explainability.model.Saliency)45 LinkedList (java.util.LinkedList)41 Value (org.kie.kogito.explainability.model.Value)41 List (java.util.List)37 LimeConfigOptimizer (org.kie.kogito.explainability.local.lime.optim.LimeConfigOptimizer)33 ValueSource (org.junit.jupiter.params.provider.ValueSource)32