Examples with ValidationRule io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.ValidationRule used on opensource projects

Example 1 with ValidationRule

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.ValidationRule in project hopsworks by logicalclocks.

the class FeatureGroupValidationsController method createOrUpdateExpectation.

public FeatureStoreExpectation createOrUpdateExpectation(Featurestore featurestore, Expectation expectation) throws FeaturestoreException {
    // Some expectation sanity checks
    Set<ValidationRule> validationRules = new HashSet<>();
    for (Rule rule : expectation.getRules()) {
        Optional<ValidationRule> validationRule = validationRuleFacade.findByName(rule.getName());
        if (!validationRule.isPresent()) {
            throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURE_STORE_RULE_NOT_FOUND, FINE, "Rule: " + rule.getName());
        }
        // Check that the rule's predicate is set
        if (validationRule.get().getPredicate() == Predicate.FEATURE && Strings.isNullOrEmpty(rule.getFeature()) || validationRule.get().getPredicate() == Predicate.ACCEPTED_TYPE && rule.getAcceptedType() == null || validationRule.get().getPredicate() == Predicate.LEGAL_VALUES && (rule.getLegalValues() == null || rule.getLegalValues().isEmpty()) || validationRule.get().getPredicate() == Predicate.PATTERN && Strings.isNullOrEmpty(rule.getPattern())) {
            throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.VALIDATION_RULE_INCOMPLETE, FINE, "Rule: " + rule.getName() + " should set " + validationRule.get().getPredicate());
        }
        if (!rule.getName().isAppliedToFeaturePairs() && rule.getMin() == null && rule.getMax() == null) {
            throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.VALIDATION_RULE_INCOMPLETE, FINE, "At least one of min/max need to be provided for rule: " + rule.getName());
        } else // Min/max is set to 1 by default for compliance rules
        if (rule.getName().isAppliedToFeaturePairs() && rule.getMin() == null && rule.getMax() == null) {
            rule.setMin(1.0);
            rule.setMax(1.0);
        }
        validationRules.add(validationRule.get());
    }
    // Persist expectation
    FeatureStoreExpectation featureStoreExpectation;
    Optional<FeatureStoreExpectation> e = featureStoreExpectationFacade.findByFeaturestoreAndName(featurestore, expectation.getName());
    featureStoreExpectation = e.orElseGet(FeatureStoreExpectation::new);
    featureStoreExpectation.setValidationRules(validationRules);
    featureStoreExpectation.setFeatureStore(featurestore);
    featureStoreExpectation.setName(expectation.getName());
    featureStoreExpectation.setDescription(expectation.getDescription());
    featureStoreExpectation.setExpectation(expectation);
    return featureStoreExpectationFacade.merge(featureStoreExpectation);
}

Example 2 with ValidationRule

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.ValidationRule in project hopsworks by logicalclocks.

the class FeatureGroupValidationsController method featureValidation.

/**
 * Validate than features exist in the feature group and that the rules can be applied to the features (types).
 *
 * @param featureStoreExpectations featureStoreExpectations
 * @param features features
 * @throws FeaturestoreException FeaturestoreException
 */
public void featureValidation(List<FeatureStoreExpectation> featureStoreExpectations, List<FeatureGroupFeatureDTO> features) throws FeaturestoreException {
    List<String> featureTypesErrMsg = new ArrayList<>();
    List<String> featureNames = new ArrayList<>();
    for (FeatureGroupFeatureDTO featureGroupFeature : features) {
        featureNames.add(featureGroupFeature.getName());
    }
    List<String> featuresNotFound = new ArrayList<>();
    for (FeatureStoreExpectation expectation : featureStoreExpectations) {
        // List to store all the features that do not exist
        for (String feature : expectation.getExpectation().getFeatures()) {
            if (!featureNames.contains(feature)) {
                featuresNotFound.add("Expectation: " + expectation.getExpectation().getName() + ", feature:" + feature);
            }
        }
        if (!featuresNotFound.isEmpty()) {
            throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURE_GROUP_EXPECTATION_FEATURE_NOT_FOUND, FINE, "expectation: " + expectation + ", feature: " + featuresNotFound);
        }
    }
    for (FeatureStoreExpectation expectation : featureStoreExpectations) {
        // Check that the data type of rules match the data types of the features
        for (FeatureGroupFeatureDTO featureGroupFeature : features) {
            if (expectation.getExpectation().getFeatures().contains(featureGroupFeature.getName())) {
                featureNames.add(featureGroupFeature.getName());
                for (ValidationRule validationRule : expectation.getValidationRules()) {
                    if (validationRule.getFeatureType() != null && !getRuleTypeMappings(validationRule.getFeatureType(), featureGroupFeature.getType())) {
                        featureTypesErrMsg.add("feature: " + featureGroupFeature.getName() + ", " + "expectation: " + expectation.getExpectation().getName() + ", " + "feature type: " + featureGroupFeature.getType() + ", " + "rule: " + validationRule.getName() + ", " + "rule type: " + validationRule.getFeatureType() + ", ");
                    }
                }
            }
        }
    }
    if (!featureTypesErrMsg.isEmpty()) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURE_GROUP_EXPECTATION_FEATURE_TYPE_INVALID, FINE, featureTypesErrMsg.toString());
    }
}