Examples with Prediction org.kie.kogito.explainability.model.Prediction used on opensource projects

Example 61 with Prediction

use of org.kie.kogito.explainability.model.Prediction in project kogito-apps by kiegroup.

the class ShapKernelExplainerTest method testErrorBounds.

// given a noisy model, expect the n% confidence window to include true value roughly n% of the time
@ParameterizedTest
@ValueSource(doubles = { .001, .1, .25, .5 })
void testErrorBounds(double noise) throws InterruptedException, ExecutionException {
    for (double interval : new double[] { .95, .975, .99 }) {
        int[] testResults = new int[600];
        for (int test = 0; test < 100; test++) {
            PredictionProvider model = TestUtils.getNoisySumModel(pc.getRandom(), noise);
            ShapConfig skConfig = testConfig.withBackground(createPIFromMatrix(backgroundAllZeros)).withConfidence(interval).build();
            List<PredictionInput> toExplain = createPIFromMatrix(toExplainAllOnes);
            ShapKernelExplainer ske = new ShapKernelExplainer(skConfig);
            List<PredictionOutput> predictionOutputs = model.predictAsync(toExplain).get();
            Prediction p = new SimplePrediction(toExplain.get(0), predictionOutputs.get(0));
            Saliency[] saliencies = ske.explainAsync(p, model).get().getSaliencies();
            RealMatrix[] explanationsAndConfs = saliencyToMatrix(saliencies);
            RealMatrix explanations = explanationsAndConfs[0];
            RealMatrix confidence = explanationsAndConfs[1];
            for (int i = 0; i < explanations.getRowDimension(); i++) {
                for (int j = 0; j < explanations.getColumnDimension(); j++) {
                    double conf = confidence.getEntry(i, j);
                    double exp = explanations.getEntry(i, j);
                    // see if true value falls into confidence interval
                    testResults[test * 6 + j] = (exp + conf) > 1.0 & 1.0 > (exp - conf) ? 1 : 0;
                }
            }
        }
        // roughly interval% of the tests should be true
        double score = Arrays.stream(testResults).sum() / 600.;
        assertEquals(interval, score, .05);
    }
}

Example 62 with Prediction

use of org.kie.kogito.explainability.model.Prediction in project kogito-apps by kiegroup.

the class ShapKernelExplainerTest method testManyFeatureRegularization2.

@Test
void testManyFeatureRegularization2() throws ExecutionException, InterruptedException {
    RealVector modelWeights = MatrixUtils.createRealMatrix(generateN(1, 25, "5021")).getRowVector(0);
    PredictionProvider model = TestUtils.getLinearModel(modelWeights.toArray());
    RealMatrix data = MatrixUtils.createRealMatrix(generateN(101, 25, "8629"));
    List<PredictionInput> toExplain = createPIFromMatrix(data.getRowMatrix(100).getData());
    List<PredictionOutput> predictionOutputs = model.predictAsync(toExplain).get();
    RealVector predictionOutputVector = MatrixUtilsExtensions.vectorFromPredictionOutput(predictionOutputs.get(0));
    Prediction p = new SimplePrediction(toExplain.get(0), predictionOutputs.get(0));
    List<PredictionInput> bg = createPIFromMatrix(new double[100][25]);
    ShapConfig sk = testConfig.copy().withBackground(bg).withRegularizer(ShapConfig.RegularizerType.AIC).withNSamples(5000).build();
    ShapKernelExplainer ske = new ShapKernelExplainer(sk);
    ShapResults shapResults = ske.explainAsync(p, model).get();
    Saliency[] saliencies = shapResults.getSaliencies();
    RealMatrix[] explanationsAndConfs = saliencyToMatrix(saliencies);
    RealMatrix explanations = explanationsAndConfs[0];
    double actualOut = predictionOutputVector.getEntry(0);
    double predOut = MatrixUtilsExtensions.sum(explanations.getRowVector(0)) + shapResults.getFnull().getEntry(0);
    assertTrue(Math.abs(predOut - actualOut) < 1e-6);
    double coefMSE = (data.getRowVector(100).ebeMultiply(modelWeights)).getDistance(explanations.getRowVector(0));
    assertTrue(coefMSE < .01);
}

Example 63 with Prediction

use of org.kie.kogito.explainability.model.Prediction in project kogito-apps by kiegroup.

the class ExplainabilityMetricsTest method testFidelityWithTextClassifier.

@Test
void testFidelityWithTextClassifier() throws ExecutionException, InterruptedException, TimeoutException {
    List<Pair<Saliency, Prediction>> pairs = new LinkedList<>();
    LimeConfig limeConfig = new LimeConfig().withSamples(10);
    LimeExplainer limeExplainer = new LimeExplainer(limeConfig);
    PredictionProvider model = TestUtils.getDummyTextClassifier();
    List<Feature> features = new LinkedList<>();
    features.add(FeatureFactory.newFulltextFeature("f-0", "brown fox", s -> Arrays.asList(s.split(" "))));
    features.add(FeatureFactory.newTextFeature("f-1", "money"));
    PredictionInput input = new PredictionInput(features);
    Prediction prediction = new SimplePrediction(input, model.predictAsync(List.of(input)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit()).get(0));
    Map<String, Saliency> saliencyMap = limeExplainer.explainAsync(prediction, model).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    for (Saliency saliency : saliencyMap.values()) {
        pairs.add(Pair.of(saliency, prediction));
    }
    Assertions.assertDoesNotThrow(() -> {
        ExplainabilityMetrics.classificationFidelity(pairs);
    });
}

Example 64 with Prediction

use of org.kie.kogito.explainability.model.Prediction in project kogito-apps by kiegroup.

the class PmmlScorecardCategoricalLimeExplainerTest method testExplanationImpactScoreWithOptimization.

@Test
void testExplanationImpactScoreWithOptimization() throws ExecutionException, InterruptedException {
    PredictionProvider model = getModel();
    List<PredictionInput> samples = getSamples();
    List<PredictionOutput> predictionOutputs = model.predictAsync(samples.subList(0, 5)).get();
    List<Prediction> predictions = DataUtils.getPredictions(samples, predictionOutputs);
    long seed = 0;
    LimeConfigOptimizer limeConfigOptimizer = new LimeConfigOptimizer().withDeterministicExecution(true).forImpactScore();
    Random random = new Random();
    PerturbationContext perturbationContext = new PerturbationContext(seed, random, 1);
    LimeConfig initialConfig = new LimeConfig().withSamples(10).withPerturbationContext(perturbationContext);
    LimeConfig optimizedConfig = limeConfigOptimizer.optimize(initialConfig, predictions, model);
    assertThat(optimizedConfig).isNotSameAs(initialConfig);
}

Example 65 with Prediction

use of org.kie.kogito.explainability.model.Prediction in project kogito-apps by kiegroup.

the class PmmlScorecardCategoricalLimeExplainerTest method testExplanationWeightedStabilityWithOptimization.

@Test
void testExplanationWeightedStabilityWithOptimization() throws ExecutionException, InterruptedException, TimeoutException {
    PredictionProvider model = getModel();
    List<PredictionInput> samples = getSamples();
    List<PredictionOutput> predictionOutputs = model.predictAsync(samples.subList(0, 5)).get();
    List<Prediction> predictions = DataUtils.getPredictions(samples, predictionOutputs);
    long seed = 0;
    LimeConfigOptimizer limeConfigOptimizer = new LimeConfigOptimizer().withDeterministicExecution(true).withWeightedStability(0.4, 0.6);
    Random random = new Random();
    PerturbationContext perturbationContext = new PerturbationContext(seed, random, 1);
    LimeConfig initialConfig = new LimeConfig().withSamples(10).withPerturbationContext(perturbationContext);
    LimeConfig optimizedConfig = limeConfigOptimizer.optimize(initialConfig, predictions, model);
    assertThat(optimizedConfig).isNotSameAs(initialConfig);
    LimeExplainer limeExplainer = new LimeExplainer(optimizedConfig);
    PredictionInput testPredictionInput = getTestInput();
    List<PredictionOutput> testPredictionOutputs = model.predictAsync(List.of(testPredictionInput)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    Prediction instance = new SimplePrediction(testPredictionInput, testPredictionOutputs.get(0));
    assertDoesNotThrow(() -> ValidationUtils.validateLocalSaliencyStability(model, instance, limeExplainer, 1, 0.5, 0.7));
}