Examples with PredictionOutput org.kie.kogito.explainability.model.PredictionOutput used on opensource projects

Example 96 with PredictionOutput

use of org.kie.kogito.explainability.model.PredictionOutput in project kogito-apps by kiegroup.

the class PartialDependencePlotExplainerTest method getMetadata.

private PredictionProviderMetadata getMetadata(Random random) {
    return new PredictionProviderMetadata() {

        @Override
        public DataDistribution getDataDistribution() {
            return DataUtils.generateRandomDataDistribution(3, 100, random);
        }

        @Override
        public PredictionInput getInputShape() {
            List<Feature> features = new LinkedList<>();
            features.add(FeatureFactory.newNumericalFeature("f0", 0));
            features.add(FeatureFactory.newNumericalFeature("f1", 0));
            features.add(FeatureFactory.newNumericalFeature("f2", 0));
            return new PredictionInput(features);
        }

        @Override
        public PredictionOutput getOutputShape() {
            List<Output> outputs = new LinkedList<>();
            outputs.add(new Output("sum-but0", Type.BOOLEAN, new Value(false), 0d));
            return new PredictionOutput(outputs);
        }
    };
}

Example 97 with PredictionOutput

use of org.kie.kogito.explainability.model.PredictionOutput in project kogito-apps by kiegroup.

the class PartialDependencePlotExplainerTest method testTextClassifier.

@ParameterizedTest
@ValueSource(ints = { 0, 1, 2, 3, 4 })
void testTextClassifier(int seed) throws Exception {
    Random random = new Random();
    random.setSeed(seed);
    PartialDependencePlotExplainer partialDependencePlotExplainer = new PartialDependencePlotExplainer();
    PredictionProvider model = TestUtils.getDummyTextClassifier();
    Collection<Prediction> predictions = new ArrayList<>(3);
    List<String> texts = List.of("we want your money", "please reply quickly", "you are the lucky winner", "huge donation for you!", "bitcoin for you");
    for (String text : texts) {
        List<Feature> features = new ArrayList<>();
        features.add(FeatureFactory.newFulltextFeature("text", text));
        PredictionInput predictionInput = new PredictionInput(features);
        PredictionOutput predictionOutput = model.predictAsync(List.of(predictionInput)).get().get(0);
        predictions.add(new SimplePrediction(predictionInput, predictionOutput));
    }
    List<PartialDependenceGraph> pdps = partialDependencePlotExplainer.explainFromPredictions(model, predictions);
    assertThat(pdps).isNotEmpty();
}

Example 98 with PredictionOutput

use of org.kie.kogito.explainability.model.PredictionOutput in project kogito-apps by kiegroup.

the class CounterfactualExplainerTest method testConsumers.

@ParameterizedTest
@ValueSource(ints = { 0, 1, 2 })
void testConsumers(int seed) throws ExecutionException, InterruptedException, TimeoutException {
    Random random = new Random();
    random.setSeed(seed);
    final List<Output> goal = List.of(new Output("inside", Type.BOOLEAN, new Value(true), 0.0));
    List<Feature> features = new LinkedList<>();
    features.add(FeatureFactory.newNumericalFeature("f-num1", 100.0, NumericalFeatureDomain.create(0.0, 1000.0)));
    features.add(FeatureFactory.newNumericalFeature("f-num2", 100.0, NumericalFeatureDomain.create(0.0, 1000.0)));
    features.add(FeatureFactory.newNumericalFeature("f-num3", 100.0, NumericalFeatureDomain.create(0.0, 1000.0)));
    features.add(FeatureFactory.newNumericalFeature("f-num4", 100.0, NumericalFeatureDomain.create(0.0, 1000.0)));
    final TerminationConfig terminationConfig = new TerminationConfig().withScoreCalculationCountLimit(10_000L);
    // for the purpose of this test, only a few steps are necessary
    final SolverConfig solverConfig = SolverConfigBuilder.builder().withTerminationConfig(terminationConfig).build();
    solverConfig.setRandomSeed((long) seed);
    solverConfig.setEnvironmentMode(EnvironmentMode.REPRODUCIBLE);
    @SuppressWarnings("unchecked") final Consumer<CounterfactualResult> assertIntermediateCounterfactualNotNull = mock(Consumer.class);
    final CounterfactualConfig counterfactualConfig = new CounterfactualConfig().withSolverConfig(solverConfig).withGoalThreshold(0.01);
    final CounterfactualExplainer counterfactualExplainer = new CounterfactualExplainer(counterfactualConfig);
    PredictionInput input = new PredictionInput(features);
    final double center = 500.0;
    final double epsilon = 10.0;
    final PredictionProvider model = TestUtils.getSumThresholdModel(center, epsilon);
    PredictionOutput output = new PredictionOutput(goal);
    Prediction prediction = new CounterfactualPrediction(input, output, null, UUID.randomUUID(), null);
    final CounterfactualResult counterfactualResult = counterfactualExplainer.explainAsync(prediction, model, assertIntermediateCounterfactualNotNull).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    for (CounterfactualEntity entity : counterfactualResult.getEntities()) {
        logger.debug("Entity: {}", entity);
    }
    logger.debug("Outputs: {}", counterfactualResult.getOutput().get(0).getOutputs());
    // At least one intermediate result is generated
    verify(assertIntermediateCounterfactualNotNull, atLeast(1)).accept(any());
}

Example 99 with PredictionOutput

use of org.kie.kogito.explainability.model.PredictionOutput in project kogito-apps by kiegroup.

the class CounterfactualExplainerTest method testNonEmptyInput.

@ParameterizedTest
@ValueSource(ints = { 0, 1, 2 })
void testNonEmptyInput(int seed) throws ExecutionException, InterruptedException, TimeoutException {
    Random random = new Random();
    random.setSeed(seed);
    final List<Output> goal = List.of(new Output("class", Type.NUMBER, new Value(10.0), 0.0d));
    List<Feature> features = new LinkedList<>();
    for (int i = 0; i < 4; i++) {
        features.add(FeatureFactory.newNumericalFeature("f-" + i, random.nextDouble(), NumericalFeatureDomain.create(0.0, 1000.0)));
    }
    final TerminationConfig terminationConfig = new TerminationConfig().withScoreCalculationCountLimit(10L);
    // for the purpose of this test, only a few steps are necessary
    final SolverConfig solverConfig = SolverConfigBuilder.builder().withTerminationConfig(terminationConfig).build();
    solverConfig.setRandomSeed((long) seed);
    solverConfig.setEnvironmentMode(EnvironmentMode.REPRODUCIBLE);
    final CounterfactualConfig counterfactualConfig = new CounterfactualConfig().withSolverConfig(solverConfig);
    final CounterfactualExplainer counterfactualExplainer = new CounterfactualExplainer(counterfactualConfig);
    PredictionProvider model = TestUtils.getSumSkipModel(0);
    PredictionInput input = new PredictionInput(features);
    PredictionOutput output = new PredictionOutput(goal);
    Prediction prediction = new CounterfactualPrediction(input, output, null, UUID.randomUUID(), null);
    final CounterfactualResult counterfactualResult = counterfactualExplainer.explainAsync(prediction, model).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    for (CounterfactualEntity entity : counterfactualResult.getEntities()) {
        logger.debug("Entity: {}", entity);
    }
    logger.debug("Outputs: {}", counterfactualResult.getOutput().get(0).getOutputs());
    assertNotNull(counterfactualResult);
    assertNotNull(counterfactualResult.getEntities());
}

Example 100 with PredictionOutput

use of org.kie.kogito.explainability.model.PredictionOutput in project kogito-apps by kiegroup.

the class CounterfactualExplainerTest method testCounterfactualMatchThreshold.

@ParameterizedTest
@ValueSource(ints = { 0, 1, 2 })
void testCounterfactualMatchThreshold(int seed) throws ExecutionException, InterruptedException, TimeoutException {
    Random random = new Random();
    random.setSeed(seed);
    final double scoreThreshold = 0.9;
    final List<Output> goal = List.of(new Output("inside", Type.BOOLEAN, new Value(true), scoreThreshold));
    List<Feature> features = new LinkedList<>();
    features.add(FeatureFactory.newNumericalFeature("f-num1", 100.0, NumericalFeatureDomain.create(0.0, 1000.0)));
    features.add(FeatureFactory.newNumericalFeature("f-num2", 100.0, NumericalFeatureDomain.create(0.0, 1000.0)));
    features.add(FeatureFactory.newNumericalFeature("f-num3", 100.0, NumericalFeatureDomain.create(0.0, 1000.0)));
    features.add(FeatureFactory.newNumericalFeature("f-num4", 100.0, NumericalFeatureDomain.create(0.0, 1000.0)));
    final double center = 500.0;
    final double epsilon = 10.0;
    final PredictionProvider model = TestUtils.getSumThresholdModel(center, epsilon);
    final CounterfactualResult result = runCounterfactualSearch((long) seed, goal, features, model, DEFAULT_GOAL_THRESHOLD);
    final List<CounterfactualEntity> counterfactualEntities = result.getEntities();
    double totalSum = 0;
    for (CounterfactualEntity entity : counterfactualEntities) {
        totalSum += entity.asFeature().getValue().asNumber();
        logger.debug("Entity: {}", entity);
    }
    assertTrue(totalSum <= center + epsilon);
    assertTrue(totalSum >= center - epsilon);
    final List<Feature> cfFeatures = counterfactualEntities.stream().map(CounterfactualEntity::asFeature).collect(Collectors.toList());
    final PredictionInput cfInput = new PredictionInput(cfFeatures);
    final PredictionOutput cfOutput = model.predictAsync(List.of(cfInput)).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit()).get(0);
    final double predictionScore = cfOutput.getOutputs().get(0).getScore();
    logger.debug("Prediction score: {}", predictionScore);
    assertTrue(predictionScore >= scoreThreshold);
    assertTrue(result.isValid());
}