Search in sources :

Example 1 with DmnDecisionModel

use of org.kie.kogito.dmn.DmnDecisionModel in project kogito-apps by kiegroup.

the class DummyDmnModelsLimeExplainerTest method testAllTypesDMNExplanation.

@Test
void testAllTypesDMNExplanation() throws ExecutionException, InterruptedException, TimeoutException {
    DMNRuntime dmnRuntime = DMNKogito.createGenericDMNRuntime(new InputStreamReader(getClass().getResourceAsStream("/dmn/allTypes.dmn")));
    assertThat(dmnRuntime.getModels().size()).isEqualTo(1);
    final String namespace = "https://kiegroup.org/dmn/_24B9EC8C-2F02-40EB-B6BB-E8CDE82FBF08";
    final String name = "new-file";
    DecisionModel decisionModel = new DmnDecisionModel(dmnRuntime, namespace, name);
    PredictionProvider model = new DecisionModelWrapper(decisionModel);
    Map<String, Object> context = new HashMap<>();
    context.put("stringInput", "test");
    context.put("listOfStringInput", Collections.singletonList("test"));
    context.put("numberInput", 1);
    context.put("listOfNumbersInput", Collections.singletonList(1));
    context.put("booleanInput", true);
    context.put("listOfBooleansInput", Collections.singletonList(true));
    context.put("timeInput", "h09:00");
    context.put("dateInput", "2020-04-02");
    context.put("dateAndTimeInput", "2020-04-02T09:00:00");
    context.put("daysAndTimeDurationInput", "P1DT1H");
    context.put("yearsAndMonthDurationInput", "P1Y1M");
    Map<String, Object> complexInput = new HashMap<>();
    complexInput.put("aNestedListOfNumbers", Collections.singletonList(1));
    complexInput.put("aNestedString", "test");
    complexInput.put("aNestedComplexInput", Collections.singletonMap("doubleNestedNumber", 1));
    context.put("complexInput", complexInput);
    context.put("listOfComplexInput", Collections.singletonList(complexInput));
    List<Feature> features = new ArrayList<>();
    features.add(FeatureFactory.newCompositeFeature("context", context));
    PredictionInput predictionInput = new PredictionInput(features);
    List<PredictionOutput> predictionOutputs = model.predictAsync(List.of(predictionInput)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    Prediction prediction = new SimplePrediction(predictionInput, predictionOutputs.get(0));
    Random random = new Random();
    PerturbationContext perturbationContext = new PerturbationContext(0L, random, 3);
    LimeConfig limeConfig = new LimeConfig().withSamples(10).withPerturbationContext(perturbationContext);
    LimeExplainer limeExplainer = new LimeExplainer(limeConfig);
    Map<String, Saliency> saliencyMap = limeExplainer.explainAsync(prediction, model).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    for (Saliency saliency : saliencyMap.values()) {
        assertThat(saliency).isNotNull();
    }
    assertThatCode(() -> ValidationUtils.validateLocalSaliencyStability(model, prediction, limeExplainer, 1, 0.5, 0.2)).doesNotThrowAnyException();
    String decision = "myDecision";
    List<PredictionInput> inputs = new ArrayList<>();
    for (int n = 0; n < 10; n++) {
        inputs.add(new PredictionInput(DataUtils.perturbFeatures(features, perturbationContext)));
    }
    DataDistribution distribution = new PredictionInputsDataDistribution(inputs);
    int k = 2;
    int chunkSize = 5;
    double precision = ExplainabilityMetrics.getLocalSaliencyPrecision(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(precision).isBetween(0d, 1d);
    double recall = ExplainabilityMetrics.getLocalSaliencyRecall(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(recall).isBetween(0d, 1d);
    double f1 = ExplainabilityMetrics.getLocalSaliencyF1(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(f1).isBetween(0d, 1d);
}
Also used : SimplePrediction(org.kie.kogito.explainability.model.SimplePrediction) PerturbationContext(org.kie.kogito.explainability.model.PerturbationContext) HashMap(java.util.HashMap) LimeExplainer(org.kie.kogito.explainability.local.lime.LimeExplainer) ArrayList(java.util.ArrayList) DecisionModel(org.kie.kogito.decision.DecisionModel) DmnDecisionModel(org.kie.kogito.dmn.DmnDecisionModel) Saliency(org.kie.kogito.explainability.model.Saliency) Feature(org.kie.kogito.explainability.model.Feature) Random(java.util.Random) PredictionInputsDataDistribution(org.kie.kogito.explainability.model.PredictionInputsDataDistribution) InputStreamReader(java.io.InputStreamReader) PredictionInput(org.kie.kogito.explainability.model.PredictionInput) Prediction(org.kie.kogito.explainability.model.Prediction) SimplePrediction(org.kie.kogito.explainability.model.SimplePrediction) PredictionProvider(org.kie.kogito.explainability.model.PredictionProvider) DMNRuntime(org.kie.dmn.api.core.DMNRuntime) LimeConfig(org.kie.kogito.explainability.local.lime.LimeConfig) PredictionInputsDataDistribution(org.kie.kogito.explainability.model.PredictionInputsDataDistribution) DataDistribution(org.kie.kogito.explainability.model.DataDistribution) PredictionOutput(org.kie.kogito.explainability.model.PredictionOutput) DmnDecisionModel(org.kie.kogito.dmn.DmnDecisionModel) Test(org.junit.jupiter.api.Test)

Example 2 with DmnDecisionModel

use of org.kie.kogito.dmn.DmnDecisionModel in project kogito-apps by kiegroup.

the class DummyDmnModelsLimeExplainerTest method testFunctional1DMNExplanation.

@Test
void testFunctional1DMNExplanation() throws ExecutionException, InterruptedException, TimeoutException {
    DMNRuntime dmnRuntime = DMNKogito.createGenericDMNRuntime(new InputStreamReader(getClass().getResourceAsStream("/dmn/functionalTest1.dmn")));
    assertThat(dmnRuntime.getModels().size()).isEqualTo(1);
    final String namespace = "https://kiegroup.org/dmn/_049CD980-1310-4B02-9E90-EFC57059F44A";
    final String name = "functionalTest1";
    DecisionModel decisionModel = new DmnDecisionModel(dmnRuntime, namespace, name);
    PredictionProvider model = new DecisionModelWrapper(decisionModel);
    Map<String, Object> context = new HashMap<>();
    context.put("booleanInput", true);
    context.put("notUsedInput", 1);
    List<Feature> features = new ArrayList<>();
    features.add(FeatureFactory.newCompositeFeature("context", context));
    PredictionInput predictionInput = new PredictionInput(features);
    List<PredictionOutput> predictionOutputs = model.predictAsync(List.of(predictionInput)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    Prediction prediction = new SimplePrediction(predictionInput, predictionOutputs.get(0));
    Random random = new Random();
    PerturbationContext perturbationContext = new PerturbationContext(0L, random, 1);
    LimeConfig limeConfig = new LimeConfig().withSamples(10).withPerturbationContext(perturbationContext);
    LimeExplainer limeExplainer = new LimeExplainer(limeConfig);
    Map<String, Saliency> saliencyMap = limeExplainer.explainAsync(prediction, model).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    for (Saliency saliency : saliencyMap.values()) {
        assertThat(saliency).isNotNull();
        List<FeatureImportance> topFeatures = saliency.getPositiveFeatures(2);
        assertThat(topFeatures.isEmpty()).isFalse();
        assertThat(topFeatures.get(0).getFeature().getName()).isEqualTo("booleanInput");
    }
    assertThatCode(() -> ValidationUtils.validateLocalSaliencyStability(model, prediction, limeExplainer, 1, 0.5, 0.5)).doesNotThrowAnyException();
    String decision = "decision";
    List<PredictionInput> inputs = new ArrayList<>();
    for (int n = 0; n < 10; n++) {
        inputs.add(new PredictionInput(DataUtils.perturbFeatures(features, perturbationContext)));
    }
    DataDistribution distribution = new PredictionInputsDataDistribution(inputs);
    int k = 2;
    int chunkSize = 5;
    double precision = ExplainabilityMetrics.getLocalSaliencyPrecision(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(precision).isBetween(0d, 1d);
    double recall = ExplainabilityMetrics.getLocalSaliencyRecall(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(recall).isBetween(0d, 1d);
    double f1 = ExplainabilityMetrics.getLocalSaliencyF1(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(f1).isBetween(0d, 1d);
}
Also used : SimplePrediction(org.kie.kogito.explainability.model.SimplePrediction) PerturbationContext(org.kie.kogito.explainability.model.PerturbationContext) HashMap(java.util.HashMap) LimeExplainer(org.kie.kogito.explainability.local.lime.LimeExplainer) ArrayList(java.util.ArrayList) DecisionModel(org.kie.kogito.decision.DecisionModel) DmnDecisionModel(org.kie.kogito.dmn.DmnDecisionModel) Saliency(org.kie.kogito.explainability.model.Saliency) Feature(org.kie.kogito.explainability.model.Feature) Random(java.util.Random) PredictionInputsDataDistribution(org.kie.kogito.explainability.model.PredictionInputsDataDistribution) InputStreamReader(java.io.InputStreamReader) PredictionInput(org.kie.kogito.explainability.model.PredictionInput) Prediction(org.kie.kogito.explainability.model.Prediction) SimplePrediction(org.kie.kogito.explainability.model.SimplePrediction) PredictionProvider(org.kie.kogito.explainability.model.PredictionProvider) DMNRuntime(org.kie.dmn.api.core.DMNRuntime) LimeConfig(org.kie.kogito.explainability.local.lime.LimeConfig) FeatureImportance(org.kie.kogito.explainability.model.FeatureImportance) PredictionInputsDataDistribution(org.kie.kogito.explainability.model.PredictionInputsDataDistribution) DataDistribution(org.kie.kogito.explainability.model.DataDistribution) PredictionOutput(org.kie.kogito.explainability.model.PredictionOutput) DmnDecisionModel(org.kie.kogito.dmn.DmnDecisionModel) Test(org.junit.jupiter.api.Test)

Example 3 with DmnDecisionModel

use of org.kie.kogito.dmn.DmnDecisionModel in project kogito-apps by kiegroup.

the class DummyDmnModelsLimeExplainerTest method testFunctional2DMNExplanation.

@Test
void testFunctional2DMNExplanation() throws ExecutionException, InterruptedException, TimeoutException {
    DMNRuntime dmnRuntime = DMNKogito.createGenericDMNRuntime(new InputStreamReader(getClass().getResourceAsStream("/dmn/functionalTest2.dmn")));
    assertThat(dmnRuntime.getModels().size()).isEqualTo(1);
    final String namespace = "https://kiegroup.org/dmn/_049CD980-1310-4B02-9E90-EFC57059F44A";
    final String name = "new-file";
    DecisionModel decisionModel = new DmnDecisionModel(dmnRuntime, namespace, name);
    PredictionProvider model = new DecisionModelWrapper(decisionModel);
    Map<String, Object> context = new HashMap<>();
    context.put("numberInput", 1);
    context.put("notUsedInput", 1);
    List<Feature> features = new ArrayList<>();
    features.add(FeatureFactory.newCompositeFeature("context", context));
    PredictionInput predictionInput = new PredictionInput(features);
    List<PredictionOutput> predictionOutputs = model.predictAsync(List.of(predictionInput)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    Prediction prediction = new SimplePrediction(predictionInput, predictionOutputs.get(0));
    Random random = new Random();
    PerturbationContext perturbationContext = new PerturbationContext(0L, random, 1);
    LimeConfig limeConfig = new LimeConfig().withSamples(10).withPerturbationContext(perturbationContext);
    LimeExplainer limeExplainer = new LimeExplainer(limeConfig);
    Map<String, Saliency> saliencyMap = limeExplainer.explainAsync(prediction, model).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    for (Saliency saliency : saliencyMap.values()) {
        assertThat(saliency).isNotNull();
        List<FeatureImportance> topFeatures = saliency.getPositiveFeatures(2);
        assertThat(topFeatures.isEmpty()).isFalse();
        assertThat(topFeatures.get(0).getFeature().getName()).isEqualTo("numberInput");
    }
    assertThatCode(() -> ValidationUtils.validateLocalSaliencyStability(model, prediction, limeExplainer, 1, 0.5, 0.5)).doesNotThrowAnyException();
    String decision = "decision";
    List<PredictionInput> inputs = new ArrayList<>();
    for (int n = 0; n < 10; n++) {
        inputs.add(new PredictionInput(DataUtils.perturbFeatures(features, perturbationContext)));
    }
    DataDistribution distribution = new PredictionInputsDataDistribution(inputs);
    int k = 2;
    int chunkSize = 5;
    double precision = ExplainabilityMetrics.getLocalSaliencyPrecision(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(precision).isBetween(0d, 1d);
    double recall = ExplainabilityMetrics.getLocalSaliencyRecall(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(recall).isBetween(0d, 1d);
    double f1 = ExplainabilityMetrics.getLocalSaliencyF1(decision, model, limeExplainer, distribution, k, chunkSize);
    assertThat(f1).isBetween(0d, 1d);
}
Also used : SimplePrediction(org.kie.kogito.explainability.model.SimplePrediction) PerturbationContext(org.kie.kogito.explainability.model.PerturbationContext) HashMap(java.util.HashMap) LimeExplainer(org.kie.kogito.explainability.local.lime.LimeExplainer) ArrayList(java.util.ArrayList) DecisionModel(org.kie.kogito.decision.DecisionModel) DmnDecisionModel(org.kie.kogito.dmn.DmnDecisionModel) Saliency(org.kie.kogito.explainability.model.Saliency) Feature(org.kie.kogito.explainability.model.Feature) Random(java.util.Random) PredictionInputsDataDistribution(org.kie.kogito.explainability.model.PredictionInputsDataDistribution) InputStreamReader(java.io.InputStreamReader) PredictionInput(org.kie.kogito.explainability.model.PredictionInput) Prediction(org.kie.kogito.explainability.model.Prediction) SimplePrediction(org.kie.kogito.explainability.model.SimplePrediction) PredictionProvider(org.kie.kogito.explainability.model.PredictionProvider) DMNRuntime(org.kie.dmn.api.core.DMNRuntime) LimeConfig(org.kie.kogito.explainability.local.lime.LimeConfig) FeatureImportance(org.kie.kogito.explainability.model.FeatureImportance) PredictionInputsDataDistribution(org.kie.kogito.explainability.model.PredictionInputsDataDistribution) DataDistribution(org.kie.kogito.explainability.model.DataDistribution) PredictionOutput(org.kie.kogito.explainability.model.PredictionOutput) DmnDecisionModel(org.kie.kogito.dmn.DmnDecisionModel) Test(org.junit.jupiter.api.Test)

Example 4 with DmnDecisionModel

use of org.kie.kogito.dmn.DmnDecisionModel in project kogito-apps by kiegroup.

the class FraudScoringDmnLimeExplainerTest method getModel.

private PredictionProvider getModel() {
    DMNRuntime dmnRuntime = DMNKogito.createGenericDMNRuntime(new InputStreamReader(getClass().getResourceAsStream("/dmn/fraud.dmn")));
    assertEquals(1, dmnRuntime.getModels().size());
    final String FRAUD_NS = "http://www.redhat.com/dmn/definitions/_81556584-7d78-4f8c-9d5f-b3cddb9b5c73";
    final String FRAUD_NAME = "fraud-scoring";
    DecisionModel decisionModel = new DmnDecisionModel(dmnRuntime, FRAUD_NS, FRAUD_NAME);
    return new DecisionModelWrapper(decisionModel, List.of("Last Transaction", "Merchant Blacklist"));
}
Also used : InputStreamReader(java.io.InputStreamReader) DmnDecisionModel(org.kie.kogito.dmn.DmnDecisionModel) DecisionModel(org.kie.kogito.decision.DecisionModel) DmnDecisionModel(org.kie.kogito.dmn.DmnDecisionModel) DMNRuntime(org.kie.dmn.api.core.DMNRuntime)

Example 5 with DmnDecisionModel

use of org.kie.kogito.dmn.DmnDecisionModel in project kogito-apps by kiegroup.

the class FraudScoringDmnPDPExplainerTest method testFraudScoringDMNExplanation.

@Test
void testFraudScoringDMNExplanation() throws ExecutionException, InterruptedException, TimeoutException {
    DMNRuntime dmnRuntime = DMNKogito.createGenericDMNRuntime(new InputStreamReader(getClass().getResourceAsStream("/dmn/fraud.dmn")));
    assertEquals(1, dmnRuntime.getModels().size());
    final String FRAUD_NS = "http://www.redhat.com/dmn/definitions/_81556584-7d78-4f8c-9d5f-b3cddb9b5c73";
    final String FRAUD_NAME = "fraud-scoring";
    DecisionModel decisionModel = new DmnDecisionModel(dmnRuntime, FRAUD_NS, FRAUD_NAME);
    PredictionProvider model = new DecisionModelWrapper(decisionModel);
    List<PredictionInput> inputs = DmnTestUtils.randomFraudScoringInputs();
    List<PredictionOutput> predictionOutputs = model.predictAsync(inputs).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    List<Prediction> predictions = new ArrayList<>();
    for (int i = 0; i < predictionOutputs.size(); i++) {
        predictions.add(new SimplePrediction(inputs.get(i), predictionOutputs.get(i)));
    }
    PartialDependencePlotExplainer partialDependencePlotExplainer = new PartialDependencePlotExplainer();
    List<PartialDependenceGraph> pdps = partialDependencePlotExplainer.explainFromPredictions(model, predictions);
    assertThat(pdps).isNotNull();
    Assertions.assertThat(pdps).hasSize(32);
}
Also used : SimplePrediction(org.kie.kogito.explainability.model.SimplePrediction) InputStreamReader(java.io.InputStreamReader) PredictionInput(org.kie.kogito.explainability.model.PredictionInput) SimplePrediction(org.kie.kogito.explainability.model.SimplePrediction) Prediction(org.kie.kogito.explainability.model.Prediction) ArrayList(java.util.ArrayList) DecisionModel(org.kie.kogito.decision.DecisionModel) DmnDecisionModel(org.kie.kogito.dmn.DmnDecisionModel) PredictionProvider(org.kie.kogito.explainability.model.PredictionProvider) DMNRuntime(org.kie.dmn.api.core.DMNRuntime) PartialDependencePlotExplainer(org.kie.kogito.explainability.global.pdp.PartialDependencePlotExplainer) PredictionOutput(org.kie.kogito.explainability.model.PredictionOutput) DmnDecisionModel(org.kie.kogito.dmn.DmnDecisionModel) PartialDependenceGraph(org.kie.kogito.explainability.model.PartialDependenceGraph) Test(org.junit.jupiter.api.Test)

Aggregations

DmnDecisionModel (org.kie.kogito.dmn.DmnDecisionModel)19 DMNRuntime (org.kie.dmn.api.core.DMNRuntime)18 DecisionModel (org.kie.kogito.decision.DecisionModel)18 InputStreamReader (java.io.InputStreamReader)15 ArrayList (java.util.ArrayList)8 Test (org.junit.jupiter.api.Test)7 Prediction (org.kie.kogito.explainability.model.Prediction)7 PredictionInput (org.kie.kogito.explainability.model.PredictionInput)7 PredictionOutput (org.kie.kogito.explainability.model.PredictionOutput)7 PredictionProvider (org.kie.kogito.explainability.model.PredictionProvider)7 SimplePrediction (org.kie.kogito.explainability.model.SimplePrediction)7 PartialDependencePlotExplainer (org.kie.kogito.explainability.global.pdp.PartialDependencePlotExplainer)4 PartialDependenceGraph (org.kie.kogito.explainability.model.PartialDependenceGraph)4 HashMap (java.util.HashMap)3 Random (java.util.Random)3 DMNContext (org.kie.dmn.api.core.DMNContext)3 LimeConfig (org.kie.kogito.explainability.local.lime.LimeConfig)3 LimeExplainer (org.kie.kogito.explainability.local.lime.LimeExplainer)3 DataDistribution (org.kie.kogito.explainability.model.DataDistribution)3 Feature (org.kie.kogito.explainability.model.Feature)3