Examples with SalienciesResponse org.kie.kogito.trusty.service.common.responses.SalienciesResponse used on opensource projects

Example 1 with SalienciesResponse

use of org.kie.kogito.trusty.service.common.responses.SalienciesResponse in project kogito-apps by kiegroup.

the class ExplainabilityApiV1IT method testSalienciesWithExplainabilityResult.

@Test
void testSalienciesWithExplainabilityResult() {
    mockServiceWithExplainabilityResult();
    Decision decision = new Decision(TEST_EXECUTION_ID, "sourceUrl", "serviceUrl", 0L, true, "executorName", "executorModelName", "executorModelNamespace", new ArrayList<>(), new ArrayList<>());
    decision.getOutcomes().add(new DecisionOutcome("outcomeId1", "Output1", ExplainabilityStatus.SUCCEEDED.name(), new UnitValue("type", new IntNode(1)), Collections.emptyList(), Collections.emptyList()));
    decision.getOutcomes().add(new DecisionOutcome("outcomeId2", "Output2", ExplainabilityStatus.SUCCEEDED.name(), new UnitValue("type2", new IntNode(2)), Collections.emptyList(), Collections.emptyList()));
    when(executionService.getDecisionById(eq(TEST_EXECUTION_ID))).thenReturn(decision);
    SalienciesResponse response = given().filter(new ResponseLoggingFilter()).when().get("/executions/decisions/" + TEST_EXECUTION_ID + "/explanations/saliencies").as(SalienciesResponse.class);
    assertNotNull(response);
    assertNotNull(response.getSaliencies());
    assertSame(2, response.getSaliencies().size());
    List<SaliencyModel> sortedSaliencies = response.getSaliencies().stream().sorted((s1, s2) -> new CompareToBuilder().append(s1.getOutcomeName(), s2.getOutcomeName()).toComparison()).collect(Collectors.toList());
    assertNotNull(sortedSaliencies.get(0));
    assertEquals("Output1", sortedSaliencies.get(0).getOutcomeName());
    assertNotNull(sortedSaliencies.get(0).getFeatureImportance());
    assertSame(2, sortedSaliencies.get(0).getFeatureImportance().size());
    assertEquals("Feature1", sortedSaliencies.get(0).getFeatureImportance().get(0).getFeatureName());
    assertEquals(0.49384, sortedSaliencies.get(0).getFeatureImportance().get(0).getFeatureScore());
    assertEquals("Feature2", sortedSaliencies.get(0).getFeatureImportance().get(1).getFeatureName());
    assertEquals(-0.1084, sortedSaliencies.get(0).getFeatureImportance().get(1).getFeatureScore());
    assertNotNull(sortedSaliencies.get(1));
    assertEquals("Output2", sortedSaliencies.get(1).getOutcomeName());
    assertNotNull(sortedSaliencies.get(1).getFeatureImportance());
    assertSame(2, sortedSaliencies.get(1).getFeatureImportance().size());
    assertEquals("Feature1", sortedSaliencies.get(1).getFeatureImportance().get(0).getFeatureName());
    assertEquals(0.0, sortedSaliencies.get(1).getFeatureImportance().get(0).getFeatureScore());
    assertEquals("Feature2", sortedSaliencies.get(1).getFeatureImportance().get(1).getFeatureName());
    assertEquals(0.70293, sortedSaliencies.get(1).getFeatureImportance().get(1).getFeatureScore());
}

Example 2 with SalienciesResponse

use of org.kie.kogito.trusty.service.common.responses.SalienciesResponse in project kogito-apps by kiegroup.

the class LIMESaliencyConverterTest method testFromResult_DecisionExists.

@Test
public void testFromResult_DecisionExists() {
    LIMEExplainabilityResult result = LIMEExplainabilityResult.buildSucceeded(EXECUTION_ID, List.of(new SaliencyModel("outcomeName1", List.of(new FeatureImportanceModel("feature1a", 1.0), new FeatureImportanceModel("feature1b", 2.0))), new SaliencyModel("outcomeName2", List.of(new FeatureImportanceModel("feature2", 3.0)))));
    Decision decision = new Decision(EXECUTION_ID, "sourceUrl", "serviceUrl", 0L, true, "executorName", "executorModelName", "executorModelNamespace", new ArrayList<>(), new ArrayList<>());
    decision.getOutcomes().add(new DecisionOutcome("outcomeId1", "outcomeName1", ExplainabilityStatus.SUCCEEDED.name(), new UnitValue("type", new IntNode(1)), Collections.emptyList(), Collections.emptyList()));
    decision.getOutcomes().add(new DecisionOutcome("outcomeId2", "outcomeName2", ExplainabilityStatus.SUCCEEDED.name(), new UnitValue("type2", new IntNode(2)), Collections.emptyList(), Collections.emptyList()));
    when(trustyService.getDecisionById(eq(EXECUTION_ID))).thenReturn(decision);
    SalienciesResponse response = converter.fromResult(EXECUTION_ID, result);
    assertNotNull(response);
    assertEquals(ExplainabilityStatus.SUCCEEDED.name(), response.getStatus());
    assertEquals(2, response.getSaliencies().size());
    List<SaliencyResponse> saliencyResponses = new ArrayList<>(response.getSaliencies());
    SaliencyResponse saliencyResponse1 = saliencyResponses.get(0);
    assertEquals("outcomeId1", saliencyResponse1.getOutcomeId());
    assertEquals("outcomeName1", saliencyResponse1.getOutcomeName());
    assertEquals(2, saliencyResponse1.getFeatureImportance().size());
    Optional<FeatureImportanceModel> oFeatureImportance1Model1 = saliencyResponse1.getFeatureImportance().stream().filter(fim -> fim.getFeatureName().equals("feature1a")).findFirst();
    assertTrue(oFeatureImportance1Model1.isPresent());
    assertEquals(1.0, oFeatureImportance1Model1.get().getFeatureScore());
    Optional<FeatureImportanceModel> oFeatureImportance2Model1 = saliencyResponse1.getFeatureImportance().stream().filter(fim -> fim.getFeatureName().equals("feature1b")).findFirst();
    assertTrue(oFeatureImportance2Model1.isPresent());
    assertEquals(2.0, oFeatureImportance2Model1.get().getFeatureScore());
    SaliencyResponse saliencyResponse2 = saliencyResponses.get(1);
    assertEquals("outcomeId2", saliencyResponse2.getOutcomeId());
    assertEquals("outcomeName2", saliencyResponse2.getOutcomeName());
    assertEquals(1, saliencyResponse2.getFeatureImportance().size());
    Optional<FeatureImportanceModel> oFeatureImportance1Model2 = saliencyResponse2.getFeatureImportance().stream().filter(fim -> fim.getFeatureName().equals("feature2")).findFirst();
    assertTrue(oFeatureImportance1Model2.isPresent());
    assertEquals(3.0, oFeatureImportance1Model2.get().getFeatureScore());
}

Example 3 with SalienciesResponse

use of org.kie.kogito.trusty.service.common.responses.SalienciesResponse in project kogito-apps by kiegroup.

the class JITDMNServiceImpl method evaluateModelAndExplain.

public DMNResultWithExplanation evaluateModelAndExplain(DMNEvaluator dmnEvaluator, Map<String, Object> context) {
    LocalDMNPredictionProvider localDMNPredictionProvider = new LocalDMNPredictionProvider(dmnEvaluator);
    DMNResult dmnResult = dmnEvaluator.evaluate(context);
    Prediction prediction = new SimplePrediction(LocalDMNPredictionProvider.toPredictionInput(context), LocalDMNPredictionProvider.toPredictionOutput(dmnResult));
    LimeConfig limeConfig = new LimeConfig().withSamples(explainabilityLimeSampleSize).withPerturbationContext(new PerturbationContext(new Random(), explainabilityLimeNoOfPerturbation));
    LimeExplainer limeExplainer = new LimeExplainer(limeConfig);
    Map<String, Saliency> saliencyMap;
    try {
        saliencyMap = limeExplainer.explainAsync(prediction, localDMNPredictionProvider).get(Config.INSTANCE.getAsyncTimeout(), Config.INSTANCE.getAsyncTimeUnit());
    } catch (TimeoutException | InterruptedException | ExecutionException e) {
        if (e instanceof InterruptedException) {
            LOGGER.error("Critical InterruptedException occurred", e);
            Thread.currentThread().interrupt();
        }
        return new DMNResultWithExplanation(new JITDMNResult(dmnEvaluator.getNamespace(), dmnEvaluator.getName(), dmnResult), new SalienciesResponse(EXPLAINABILITY_FAILED, EXPLAINABILITY_FAILED_MESSAGE, null));
    }
    List<SaliencyResponse> saliencyModelResponse = buildSalienciesResponse(dmnEvaluator.getDmnModel(), saliencyMap);
    return new DMNResultWithExplanation(new JITDMNResult(dmnEvaluator.getNamespace(), dmnEvaluator.getName(), dmnResult), new SalienciesResponse(EXPLAINABILITY_SUCCEEDED, null, saliencyModelResponse));
}

Example 4 with SalienciesResponse

use of org.kie.kogito.trusty.service.common.responses.SalienciesResponse in project kogito-apps by kiegroup.

the class AbstractTrustyExplainabilityEnd2EndIT method doTest.

@Test
public void doTest() {
    try (final Network network = Network.newNetwork();
        final KogitoInfinispanContainer infinispan = new KogitoInfinispanContainer().withNetwork(network).withNetworkAliases(INFINISPAN_ALIAS);
        final KogitoKafkaContainer kafka = new KogitoKafkaContainer().withNetwork(network).withNetworkAliases(KAFKA_ALIAS);
        final KogitoKeycloakContainer keycloak = new KogitoKeycloakContainer().withEnv(KEYCLOAK_DB_VENDOR_VARIABLE, KEYCLOAK_DB_VENDOR_VALUE).withNetwork(network).withNetworkAliases(KEYCLOAK_ALIAS);
        final ExplainabilityServiceMessagingContainer explService = new ExplainabilityServiceMessagingContainer(KAFKA_BOOTSTRAP_SERVERS, EXPL_SERVICE_SAMPLES).withLogConsumer(new Slf4jLogConsumer(LOGGER)).withNetwork(network).withNetworkAliases(EXPL_SERVICE_ALIAS);
        final InfinispanTrustyServiceContainer trustyService = new InfinispanTrustyServiceContainer(INFINISPAN_SERVER_LIST, KAFKA_BOOTSTRAP_SERVERS, true).withEnv(TRUSTY_SERVICE_OIDC_AUTH_SERVER_URL_VARIABLE, TRUSTY_SERVICE_OIDC_AUTH_SERVER_URL_VALUE).withEnv(TRUSTY_SERVICE_OIDC_CLIENT_ID_VARIABLE, TRUSTY_SERVICE_OIDC_CLIENT_ID_VALUE).withEnv("INFINISPAN_USE_AUTH", "FALSE").withLogConsumer(new Slf4jLogConsumer(LOGGER)).withNetwork(network).withNetworkAliases(TRUSTY_SERVICE_ALIAS);
        final KogitoServiceContainer kogitoService = kogitoServiceContainerProducer.apply(KAFKA_BOOTSTRAP_SERVERS, KOGITO_SERVICE_URL).withLogConsumer(new Slf4jLogConsumer(LOGGER)).withNetwork(network).withNetworkAliases(KOGITO_SERVICE_ALIAS)) {
        infinispan.start();
        assertTrue(infinispan.isRunning());
        kafka.start();
        assertTrue(kafka.isRunning());
        keycloak.start();
        assertTrue(keycloak.isRunning());
        explService.start();
        assertTrue(explService.isRunning());
        trustyService.start();
        assertTrue(trustyService.isRunning());
        kogitoService.start();
        assertTrue(kogitoService.isRunning());
        final String accessToken = given().port(keycloak.getFirstMappedPort()).param(KEYCLOAK_GRANT_TYPE_PARAM_NAME, KEYCLOAK_GRANT_TYPE_PARAM_VALUE).param(KEYCLOAK_USERNAME_PARAM_NAME, KEYCLOAK_USERNAME_PARAM_VALUE).param(KEYCLOAK_PASSWORD_PARAM_NAME, KEYCLOAK_PASSWORD_PARAM_VALUE).param(KEYCLOAK_CLIENT_ID_PARAM_NAME, KEYCLOAK_CLIENT_ID_PARAM_VALUE).param(KEYCLOAK_CLIENT_SECRET_PARAM_NAME, KEYCLOAK_CLIENT_SECRET_PARAM_VALUE).when().post(KEYCLOAK_ACCESS_TOKEN_PATH).as(AccessTokenResponse.class).getToken();
        assertNotNull(accessToken);
        final List<String> executionIds = new ArrayList<>();
        final int expectedExecutions = KOGITO_SERVICE_PAYLOADS.size();
        LOGGER.info("Invoke Decision endpoint to generate LIME explanations...");
        KOGITO_SERVICE_PAYLOADS.forEach(json -> given().port(kogitoService.getFirstMappedPort()).contentType("application/json").body(json).when().post("/Traffic Violation").then().statusCode(200));
        LOGGER.info("Check Decisions executed...");
        await().atLeast(5, SECONDS).atMost(30, SECONDS).with().pollInterval(5, SECONDS).untilAsserted(() -> {
            ExecutionsResponse executionsResponse = given().port(trustyService.getFirstMappedPort()).auth().oauth2(accessToken).when().get(String.format("/executions?limit=%d", expectedExecutions)).then().statusCode(200).extract().as(ExecutionsResponse.class);
            assertSame(expectedExecutions, executionsResponse.getHeaders().size());
            executionsResponse.getHeaders().forEach(h -> executionIds.add(h.getExecutionId()));
        });
        LOGGER.info("Check LIME explanations generated...");
        await().atLeast(5, SECONDS).atMost(60, SECONDS).with().pollInterval(5, SECONDS).untilAsserted(() -> {
            executionIds.forEach(executionId -> {
                SalienciesResponse salienciesResponse = given().port(trustyService.getFirstMappedPort()).auth().oauth2(accessToken).when().get("/executions/decisions/" + executionId + "/explanations/saliencies").then().statusCode(200).extract().as(SalienciesResponse.class);
                assertEquals("SUCCEEDED", salienciesResponse.getStatus());
            });
        });
        LOGGER.info("Request Counterfactuals for each execution and check responses generated...");
        executionIds.forEach(executionId -> {
            await().atLeast(500, MILLISECONDS).atMost(60, SECONDS).with().pollInterval(500, MILLISECONDS).until(doCounterfactualRequests(trustyService, accessToken, executionId));
            await().atLeast(500, MILLISECONDS).atMost(60, SECONDS).with().pollInterval(500, MILLISECONDS).until(doCounterfactualResponses(trustyService, accessToken, executionId));
        });
    }
}

Example 5 with SalienciesResponse

use of org.kie.kogito.trusty.service.common.responses.SalienciesResponse in project kogito-apps by kiegroup.

the class LIMESaliencyConverterTest method testFromResult_DecisionExists_WhenOutcomeNameNotFound.

@Test
public void testFromResult_DecisionExists_WhenOutcomeNameNotFound() {
    LIMEExplainabilityResult result = LIMEExplainabilityResult.buildSucceeded(EXECUTION_ID, List.of(new SaliencyModel("outcomeName1", List.of(new FeatureImportanceModel("feature1", 1.0))), new SaliencyModel("outcomeName2", List.of(new FeatureImportanceModel("feature2", 2.0)))));
    Decision decision = new Decision(EXECUTION_ID, "sourceUrl", "serviceUrl", 0L, true, "executorName", "executorModelName", "executorModelNamespace", new ArrayList<>(), new ArrayList<>());
    decision.getOutcomes().add(new DecisionOutcome("outcomeId1", "outcomeName1", ExplainabilityStatus.SUCCEEDED.name(), new UnitValue("type", new IntNode(1)), Collections.emptyList(), Collections.emptyList()));
    decision.getOutcomes().add(new DecisionOutcome("outcomeId2", "outcomeNameX", ExplainabilityStatus.SUCCEEDED.name(), new UnitValue("type2", new IntNode(2)), Collections.emptyList(), Collections.emptyList()));
    when(trustyService.getDecisionById(eq(EXECUTION_ID))).thenReturn(decision);
    SalienciesResponse response = converter.fromResult(EXECUTION_ID, result);
    assertNotNull(response);
    assertEquals(ExplainabilityStatus.SUCCEEDED.name(), response.getStatus());
    assertEquals(1, response.getSaliencies().size());
    List<SaliencyResponse> saliencyResponses = new ArrayList<>(response.getSaliencies());
    SaliencyResponse saliencyResponse1 = saliencyResponses.get(0);
    assertEquals("outcomeId1", saliencyResponse1.getOutcomeId());
    assertEquals("outcomeName1", saliencyResponse1.getOutcomeName());
    assertEquals(1, saliencyResponse1.getFeatureImportance().size());
    Optional<FeatureImportanceModel> oFeatureImportance1Model1 = saliencyResponse1.getFeatureImportance().stream().filter(fim -> fim.getFeatureName().equals("feature1")).findFirst();
    assertTrue(oFeatureImportance1Model1.isPresent());
    assertEquals(1.0, oFeatureImportance1Model1.get().getFeatureScore());
}