use of org.kie.kogito.explainability.local.lime.LimeExplainer in project kogito-apps by kiegroup.
the class OpenNLPLimeExplainerTest method testExplanationStabilityWithOptimization.
@Test
void testExplanationStabilityWithOptimization() throws ExecutionException, InterruptedException, TimeoutException, IOException {
PredictionProvider model = getModel();
List<PredictionInput> samples = getSamples(getTokenizer());
List<PredictionOutput> predictionOutputs = model.predictAsync(samples.subList(0, 5)).get();
List<Prediction> predictions = DataUtils.getPredictions(samples, predictionOutputs);
long seed = 0;
LimeConfigOptimizer limeConfigOptimizer = new LimeConfigOptimizer().withDeterministicExecution(true).withSampling(false).withStepCountLimit(30);
Random random = new Random();
LimeConfig limeConfig = new LimeConfig().withSamples(10).withPerturbationContext(new PerturbationContext(seed, random, 1));
LimeConfig optimizedConfig = limeConfigOptimizer.optimize(limeConfig, predictions, model);
Assertions.assertThat(optimizedConfig).isNotSameAs(limeConfig);
LimeExplainer limeExplainer = new LimeExplainer(optimizedConfig);
PredictionInput testPredictionInput = getTestInput(getTokenizer());
List<PredictionOutput> testPredictionOutputs = model.predictAsync(List.of(testPredictionInput)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
Prediction instance = new SimplePrediction(testPredictionInput, testPredictionOutputs.get(0));
assertDoesNotThrow(() -> ValidationUtils.validateLocalSaliencyStability(model, instance, limeExplainer, 1, 0.8, 0.8));
}
use of org.kie.kogito.explainability.local.lime.LimeExplainer in project kogito-apps by kiegroup.
the class PmmlCompoundScorecardLimeExplainerTest method testPMMLCompoundScorecard.
@Test
void testPMMLCompoundScorecard() throws Exception {
Random random = new Random();
LimeConfig limeConfig = new LimeConfig().withSamples(10).withPerturbationContext(new PerturbationContext(0L, random, 1));
LimeExplainer limeExplainer = new LimeExplainer(limeConfig);
PredictionInput input = getTestInput();
PredictionProvider model = getModel();
List<PredictionOutput> predictionOutputs = model.predictAsync(List.of(input)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
assertThat(predictionOutputs).isNotNull();
assertThat(predictionOutputs).isNotEmpty();
PredictionOutput output = predictionOutputs.get(0);
assertThat(output).isNotNull();
Prediction prediction = new SimplePrediction(input, output);
Map<String, Saliency> saliencyMap = limeExplainer.explainAsync(prediction, model).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
for (Saliency saliency : saliencyMap.values()) {
assertThat(saliency).isNotNull();
double v = ExplainabilityMetrics.impactScore(model, prediction, saliency.getTopFeatures(2));
assertThat(v).isEqualTo(1d);
}
assertDoesNotThrow(() -> ValidationUtils.validateLocalSaliencyStability(model, prediction, limeExplainer, 1, 0.5, 0.5));
List<PredictionInput> inputs = getSamples();
DataDistribution distribution = new PredictionInputsDataDistribution(inputs);
String decision = "score";
int k = 1;
int chunkSize = 2;
double f1 = ExplainabilityMetrics.getLocalSaliencyF1(decision, model, limeExplainer, distribution, k, chunkSize);
AssertionsForClassTypes.assertThat(f1).isBetween(0d, 1d);
}
use of org.kie.kogito.explainability.local.lime.LimeExplainer in project kogito-apps by kiegroup.
the class PmmlCompoundScorecardLimeExplainerTest method testExplanationWeightedStabilityWithOptimization.
@Test
void testExplanationWeightedStabilityWithOptimization() throws ExecutionException, InterruptedException, TimeoutException {
PredictionProvider model = getModel();
List<PredictionInput> samples = getSamples();
List<PredictionOutput> predictionOutputs = model.predictAsync(samples.subList(0, 5)).get();
List<Prediction> predictions = DataUtils.getPredictions(samples, predictionOutputs);
long seed = 0;
LimeConfigOptimizer limeConfigOptimizer = new LimeConfigOptimizer().withDeterministicExecution(true).withWeightedStability(0.4, 0.6);
Random random = new Random();
PerturbationContext perturbationContext = new PerturbationContext(seed, random, 1);
LimeConfig initialConfig = new LimeConfig().withSamples(10).withPerturbationContext(perturbationContext);
LimeConfig optimizedConfig = limeConfigOptimizer.optimize(initialConfig, predictions, model);
assertThat(optimizedConfig).isNotSameAs(initialConfig);
LimeExplainer limeExplainer = new LimeExplainer(optimizedConfig);
PredictionInput testPredictionInput = getTestInput();
List<PredictionOutput> testPredictionOutputs = model.predictAsync(List.of(testPredictionInput)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
Prediction instance = new SimplePrediction(testPredictionInput, testPredictionOutputs.get(0));
assertDoesNotThrow(() -> ValidationUtils.validateLocalSaliencyStability(model, instance, limeExplainer, 1, 0.5, 0.7));
}
use of org.kie.kogito.explainability.local.lime.LimeExplainer in project kogito-apps by kiegroup.
the class PmmlRegressionCategoricalLimeExplainerTest method testExplanationStabilityWithOptimization.
@Disabled("See KOGITO-6154")
@Test
void testExplanationStabilityWithOptimization() throws ExecutionException, InterruptedException, TimeoutException {
PredictionProvider model = getModel();
List<PredictionInput> samples = getSamples();
List<PredictionOutput> predictionOutputs = model.predictAsync(samples.subList(0, 5)).get();
List<Prediction> predictions = DataUtils.getPredictions(samples, predictionOutputs);
long seed = 0;
LimeConfigOptimizer limeConfigOptimizer = new LimeConfigOptimizer().withDeterministicExecution(true);
Random random = new Random();
PerturbationContext perturbationContext = new PerturbationContext(seed, random, 1);
LimeConfig initialConfig = new LimeConfig().withSamples(10).withPerturbationContext(perturbationContext);
LimeConfig optimizedConfig = limeConfigOptimizer.optimize(initialConfig, predictions, model);
assertThat(optimizedConfig).isNotSameAs(initialConfig);
LimeExplainer limeExplainer = new LimeExplainer(optimizedConfig);
PredictionInput testPredictionInput = getTestInput();
List<PredictionOutput> testPredictionOutputs = model.predictAsync(List.of(testPredictionInput)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
Prediction instance = new SimplePrediction(testPredictionInput, testPredictionOutputs.get(0));
assertDoesNotThrow(() -> ValidationUtils.validateLocalSaliencyStability(model, instance, limeExplainer, 1, 0.6, 0.6));
}
use of org.kie.kogito.explainability.local.lime.LimeExplainer in project kogito-apps by kiegroup.
the class ExplainabilityMetricsTest method testFidelityWithTextClassifier.
@Test
void testFidelityWithTextClassifier() throws ExecutionException, InterruptedException, TimeoutException {
List<Pair<Saliency, Prediction>> pairs = new LinkedList<>();
LimeConfig limeConfig = new LimeConfig().withSamples(10);
LimeExplainer limeExplainer = new LimeExplainer(limeConfig);
PredictionProvider model = TestUtils.getDummyTextClassifier();
List<Feature> features = new LinkedList<>();
features.add(FeatureFactory.newFulltextFeature("f-0", "brown fox", s -> Arrays.asList(s.split(" "))));
features.add(FeatureFactory.newTextFeature("f-1", "money"));
PredictionInput input = new PredictionInput(features);
Prediction prediction = new SimplePrediction(input, model.predictAsync(List.of(input)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit()).get(0));
Map<String, Saliency> saliencyMap = limeExplainer.explainAsync(prediction, model).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
for (Saliency saliency : saliencyMap.values()) {
pairs.add(Pair.of(saliency, prediction));
}
Assertions.assertDoesNotThrow(() -> {
ExplainabilityMetrics.classificationFidelity(pairs);
});
}
Aggregations