Examples with InvalidContent org.ihtsdo.drools.response.InvalidContent used on opensource projects

Example 1 with InvalidContent

use of org.ihtsdo.drools.response.InvalidContent in project snomed-drools by IHTSDO.

the class RuleExecutor method execute.

/**
 * Validate a concept using drools rules available to this executor.
 * A temporary identifier should be assigned to the concepts or any of it's descriptions and relationships
 * if the component is new and does not yet have an SCTID. The identifier of the component is used to identify invalid content.
 *
 * Passing services in with every invocation of this method allows the implementation to capture content context. For example
 * services relevant to the content branch being worked on.
 * @param ruleSetNames The rule sets to use during validation.
 * @param concepts The concepts to be validated.
 * @param conceptService An implementation of the ConceptService class for use in validation rules.
 * @param descriptionService An implementation of the DescriptionService class for use in validation rules.
 * @param relationshipService An implementation of the RelationshipService class for use in validation rules.
 * @param includePublishedComponents Include the published components of the given concept in results if found to be invalid.
 *                                   Published content will be used during validation regardless just not returned.
 * @param includeInferredRelationships Include the inferred relationships of the given concept during validation and
 *                                     in results if found to be invalid.
 * @return A list of content found to be invalid is returned.
 */
public List<InvalidContent> execute(Set<String> ruleSetNames, Collection<? extends Concept> concepts, ConceptService conceptService, DescriptionService descriptionService, RelationshipService relationshipService, boolean includePublishedComponents, boolean includeInferredRelationships) throws RuleExecutorException {
    for (Concept concept : concepts) {
        assertComponentIdsPresent(concept);
    }
    Date start = new Date();
    final List<List<InvalidContent>> sessionInvalidContent = new ArrayList<>();
    final List<InvalidContent> exceptionContents = new ArrayList<>();
    for (String ruleSetName : ruleSetNames) {
        final KieContainer kieContainer = assertionGroupContainers.get(ruleSetName);
        if (kieContainer == null) {
            throw new RuleExecutorException("Rule set not found for name '" + ruleSetName + "'");
        }
        int threads = concepts.size() == 1 ? 1 : 10;
        ExecutorService executorService = Executors.newFixedThreadPool(threads);
        List<StatelessKieSession> sessions = new ArrayList<>();
        for (int s = 0; s < threads; s++) {
            // List per thread to avoid concurrency issues.
            ArrayList<InvalidContent> invalidContent = new ArrayList<>();
            sessionInvalidContent.add(invalidContent);
            sessions.add(newStatelessKieSession(kieContainer, conceptService, descriptionService, relationshipService, invalidContent));
        }
        List<Concept> conceptList = new ArrayList<>(concepts);
        List<Callable<String>> tasks = new ArrayList<>();
        String total = String.format("%,d", concepts.size());
        int i = 0;
        while (i < concepts.size()) {
            Set<Component> components = new HashSet<>();
            Concept concept = conceptList.get(i++);
            addConcept(components, concept, includeInferredRelationships);
            int sessionIndex = tasks.size();
            tasks.add(() -> {
                try {
                    StatelessKieSession statelessKieSession = sessions.get(sessionIndex);
                    statelessKieSession.execute(components);
                    components.clear();
                    ((StatelessKnowledgeSessionImpl) statelessKieSession).newWorkingMemory();
                } catch (Exception e) {
                    exceptionContents.add(new InvalidContent(concept.getId(), concept, "An error occurred while running concept validation. Technical detail: " + e.getMessage(), Severity.ERROR));
                }
                return null;
            });
            if (tasks.size() == threads) {
                runTasks(executorService, tasks);
                tasks.clear();
            }
            if (i % 10_000 == 0) {
                logger.info("Validated {} of {}", String.format("%,d", i), total);
            }
        }
        if (!tasks.isEmpty()) {
            runTasks(executorService, tasks);
        }
        executorService.shutdown();
        logger.info("Validated {} of {}", String.format("%,d", i), total);
        logger.info("Rule execution took {} seconds", (new Date().getTime() - start.getTime()) / 1000);
    }
    List<InvalidContent> invalidContent = sessionInvalidContent.stream().flatMap(Collection::stream).filter(Objects::nonNull).collect(Collectors.toList());
    invalidContent.addAll(exceptionContents);
    invalidContent = removeDuplicates(invalidContent);
    if (!includePublishedComponents) {
        Set<InvalidContent> publishedInvalidContent = new HashSet<>();
        for (InvalidContent invalidContentItem : invalidContent) {
            logger.info("invalidContentItem : {}, {}, {}, {}, {}", invalidContentItem.getConceptId(), invalidContentItem.isIgnorePublishedCheck(), invalidContentItem.isPublished(), invalidContentItem.getSeverity(), invalidContentItem.getMessage());
            if (!invalidContentItem.isIgnorePublishedCheck() && invalidContentItem.isPublished()) {
                publishedInvalidContent.add(invalidContentItem);
            }
        }
        invalidContent.removeAll(publishedInvalidContent);
    }
    if (testResourcesEmpty) {
        invalidContent.add(0, InvalidContent.getGeneralWarning("Test resources were not available so assertions like case significance and US specific terms " + "checks will not have run."));
    }
    return invalidContent;
}

Example 2 with InvalidContent

use of org.ihtsdo.drools.response.InvalidContent in project snomed-drools by IHTSDO.

the class DroolsRF2Validator method validate.

private void validate(Set<String> assertionGroups, Set<String> extractedRF2FilesDirectories, String currentEffectiveTime, Set<String> includedModuleSets, Set<String> previousReleaseDirectories) {
    File report = new File("validation-report-" + new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss").format(new Date()) + ".txt");
    try {
        // Load test resources from public S3 location
        // Run assertions
        List<InvalidContent> invalidContents = validateRF2Files(extractedRF2FilesDirectories, previousReleaseDirectories, assertionGroups, currentEffectiveTime, includedModuleSets, false);
        // Write report
        report.createNewFile();
        try (BufferedWriter reportWriter = new BufferedWriter(new FileWriter(report))) {
            reportWriter.write("conceptId\tcomponentId\tmessage\tseverity\tignorePublishedCheck");
            reportWriter.newLine();
            for (InvalidContent invalidContent : invalidContents) {
                reportWriter.write(invalidContent.getConceptId());
                reportWriter.write(TAB);
                reportWriter.write(invalidContent.getComponentId());
                reportWriter.write(TAB);
                reportWriter.write(invalidContent.getMessage().replace("\n", " "));
                reportWriter.write(TAB);
                reportWriter.write(invalidContent.getSeverity().toString());
                reportWriter.write(TAB);
                reportWriter.write(invalidContent.isIgnorePublishedCheck() + "");
                reportWriter.newLine();
            }
        }
    } catch (ReleaseImportException | IOException e) {
        e.printStackTrace();
    }
}

Example 3 with InvalidContent

use of org.ihtsdo.drools.response.InvalidContent in project snomed-drools by IHTSDO.

the class RulesTestManual method executeRulesAndAssertExpectations.

private void executeRulesAndAssertExpectations(RuleExecutor ruleExecutor, List<TestConcept<TestDescription, TestRelationship>> conceptsToTest, boolean expectPass) {
    for (TestConcept<TestDescription, TestRelationship> concept : conceptsToTest) {
        final HashSet<String> ruleSetNames = new HashSet<>();
        ruleSetNames.add("OneRule");
        final List<InvalidContent> invalidContent = ruleExecutor.execute(ruleSetNames, Collections.singleton(concept), conceptService, descriptionService, relationshipService, false, false);
        Set<String> uniqueComponentAssertionSet = new HashSet<>();
        for (InvalidContent content : invalidContent) {
            String pair = content.getComponent().getId() + " " + content.getMessage();
            if (!uniqueComponentAssertionSet.add(pair)) {
                Assert.fail("Component failures should not be reported multiple times. Duplicate component/message found: " + pair);
            }
            // Attempt to prevent multiple assertions using the same rule id.. this does not guarantee uniqueness because not 100% test coverage.
            final String existingMessage = ruleIdToMessageMap.get(content.getRuleId());
            final String newMessage = firstPart(content.getMessage());
            if (existingMessage != null && !existingMessage.equals(newMessage)) {
                Assert.fail("Assertion id " + content.getRuleId() + " has been used with multiple failure messages: '" + existingMessage + "' AND '" + newMessage + "'");
            } else {
                ruleIdToMessageMap.put(content.getRuleId(), newMessage);
            }
        }
        if (expectPass) {
            Assert.assertEquals("A concept from the " + ASSERT_CONCEPTS_PASS + " set actually failed! " + invalidContent, 0, invalidContent.size());
        } else {
            Assert.assertNotEquals("A concept from the " + ASSERT_CONCEPTS_FAIL + " set actually passed! " + concept.toString(), 0, invalidContent.size());
        }
    }
}

Example 4 with InvalidContent

use of org.ihtsdo.drools.response.InvalidContent in project snowstorm by IHTSDO.

the class ConceptValidationHelper method replaceInvalidContentTemporaryUUIDWithSCTIDInConcept.

private static void replaceInvalidContentTemporaryUUIDWithSCTIDInConcept(final List<InvalidContent> invalidContentWarnings, final Concept concept) {
    final String conceptId = concept.getConceptId();
    final List<InvalidContent> newInvalidContentWarnings = new ArrayList<>();
    for (final Iterator<InvalidContent> iterator = invalidContentWarnings.iterator(); iterator.hasNext(); ) {
        final InvalidContent invalidContent = iterator.next();
        if (!invalidContent.getConceptId().equals(conceptId)) {
            newInvalidContentWarnings.add(new InvalidContent(invalidContent.getRuleId(), conceptId, invalidContent.getComponent(), invalidContent.getMessage(), invalidContent.getSeverity()));
            iterator.remove();
        }
    }
    invalidContentWarnings.addAll(newInvalidContentWarnings);
}

Example 5 with InvalidContent

use of org.ihtsdo.drools.response.InvalidContent in project snowstorm by IHTSDO.

the class ConceptValidationHelper method replaceTemporaryUUIDWithSCTID.

public static List<InvalidContent> replaceTemporaryUUIDWithSCTID(final List<InvalidContent> invalidContentWarnings, final Concept concept) {
    replaceInvalidContentTemporaryUUIDWithSCTIDInConcept(invalidContentWarnings, concept);
    replaceInvalidContentTemporaryUUIDWithSCTIDIn(invalidContentWarnings, concept.getDescriptions(), (final InvalidContent invalidContentWarning, final Description description) -> {
        final Component component = invalidContentWarning.getComponent();
        if (component instanceof DroolsDescription) {
            final DroolsDescription droolsDescription = (DroolsDescription) component;
            if (description != null && description.getReleaseHash() != null && description.getReleaseHash().equals(droolsDescription.getReleaseHash())) {
                invalidContentWarning.setComponent(new DroolsDescription(description));
            }
        }
    });
    replaceInvalidContentTemporaryUUIDWithSCTIDIn(invalidContentWarnings, concept.getRelationships(), (final InvalidContent invalidContentWarning, final Relationship relationship) -> {
        final Component component = invalidContentWarning.getComponent();
        if (component instanceof DroolsRelationship) {
            final DroolsRelationship droolsRelationship = (DroolsRelationship) component;
            if (relationship != null && relationship.getReleaseHash() != null && relationship.getReleaseHash().equals(droolsRelationship.getReleaseHash())) {
                invalidContentWarning.setComponent(new DroolsRelationship(null, false, relationship));
            }
        }
    });
    replaceInvalidContentTemporaryUUIDWithSCTIDInAxiom(invalidContentWarnings, concept.getClassAxioms(), false);
    replaceInvalidContentTemporaryUUIDWithSCTIDInAxiom(invalidContentWarnings, concept.getGciAxioms(), true);
    return invalidContentWarnings;
}