use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureStoreExpectation in project hopsworks by logicalclocks.
the class FeatureGroupValidationsController method attachExpectation.
public FeatureGroupExpectation attachExpectation(Featuregroup featuregroup, String name, Project project, Users user) throws FeaturestoreException {
FeatureStoreExpectation featureStoreExpectation = getFeatureStoreExpectation(featuregroup.getFeaturestore(), name);
FeatureGroupExpectation featureGroupExpectation;
Optional<FeatureGroupExpectation> attachedExpectation = featureGroupExpectationFacade.findByFeaturegroupAndExpectation(featuregroup, featureStoreExpectation);
featureValidation(featureStoreExpectation, featuregroup, project, user);
if (!attachedExpectation.isPresent()) {
featureGroupExpectation = new FeatureGroupExpectation();
featureGroupExpectation.setFeaturegroup(featuregroup);
featureGroupExpectation.setFeatureStoreExpectation(featureStoreExpectation);
return featureGroupExpectationFacade.merge(featureGroupExpectation);
} else {
return attachedExpectation.get();
}
}
use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureStoreExpectation in project hopsworks by logicalclocks.
the class FeatureGroupValidationsController method createOrUpdateExpectation.
public FeatureStoreExpectation createOrUpdateExpectation(Featurestore featurestore, Expectation expectation) throws FeaturestoreException {
// Some expectation sanity checks
Set<ValidationRule> validationRules = new HashSet<>();
for (Rule rule : expectation.getRules()) {
Optional<ValidationRule> validationRule = validationRuleFacade.findByName(rule.getName());
if (!validationRule.isPresent()) {
throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURE_STORE_RULE_NOT_FOUND, FINE, "Rule: " + rule.getName());
}
// Check that the rule's predicate is set
if (validationRule.get().getPredicate() == Predicate.FEATURE && Strings.isNullOrEmpty(rule.getFeature()) || validationRule.get().getPredicate() == Predicate.ACCEPTED_TYPE && rule.getAcceptedType() == null || validationRule.get().getPredicate() == Predicate.LEGAL_VALUES && (rule.getLegalValues() == null || rule.getLegalValues().isEmpty()) || validationRule.get().getPredicate() == Predicate.PATTERN && Strings.isNullOrEmpty(rule.getPattern())) {
throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.VALIDATION_RULE_INCOMPLETE, FINE, "Rule: " + rule.getName() + " should set " + validationRule.get().getPredicate());
}
if (!rule.getName().isAppliedToFeaturePairs() && rule.getMin() == null && rule.getMax() == null) {
throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.VALIDATION_RULE_INCOMPLETE, FINE, "At least one of min/max need to be provided for rule: " + rule.getName());
} else // Min/max is set to 1 by default for compliance rules
if (rule.getName().isAppliedToFeaturePairs() && rule.getMin() == null && rule.getMax() == null) {
rule.setMin(1.0);
rule.setMax(1.0);
}
validationRules.add(validationRule.get());
}
// Persist expectation
FeatureStoreExpectation featureStoreExpectation;
Optional<FeatureStoreExpectation> e = featureStoreExpectationFacade.findByFeaturestoreAndName(featurestore, expectation.getName());
featureStoreExpectation = e.orElseGet(FeatureStoreExpectation::new);
featureStoreExpectation.setValidationRules(validationRules);
featureStoreExpectation.setFeatureStore(featurestore);
featureStoreExpectation.setName(expectation.getName());
featureStoreExpectation.setDescription(expectation.getDescription());
featureStoreExpectation.setExpectation(expectation);
return featureStoreExpectationFacade.merge(featureStoreExpectation);
}
use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureStoreExpectation in project hopsworks by logicalclocks.
the class FeatureGroupValidationsController method featureValidation.
/**
* Validate than features exist in the feature group and that the rules can be applied to the features (types).
*
* @param featureStoreExpectations featureStoreExpectations
* @param features features
* @throws FeaturestoreException FeaturestoreException
*/
public void featureValidation(List<FeatureStoreExpectation> featureStoreExpectations, List<FeatureGroupFeatureDTO> features) throws FeaturestoreException {
List<String> featureTypesErrMsg = new ArrayList<>();
List<String> featureNames = new ArrayList<>();
for (FeatureGroupFeatureDTO featureGroupFeature : features) {
featureNames.add(featureGroupFeature.getName());
}
List<String> featuresNotFound = new ArrayList<>();
for (FeatureStoreExpectation expectation : featureStoreExpectations) {
// List to store all the features that do not exist
for (String feature : expectation.getExpectation().getFeatures()) {
if (!featureNames.contains(feature)) {
featuresNotFound.add("Expectation: " + expectation.getExpectation().getName() + ", feature:" + feature);
}
}
if (!featuresNotFound.isEmpty()) {
throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURE_GROUP_EXPECTATION_FEATURE_NOT_FOUND, FINE, "expectation: " + expectation + ", feature: " + featuresNotFound);
}
}
for (FeatureStoreExpectation expectation : featureStoreExpectations) {
// Check that the data type of rules match the data types of the features
for (FeatureGroupFeatureDTO featureGroupFeature : features) {
if (expectation.getExpectation().getFeatures().contains(featureGroupFeature.getName())) {
featureNames.add(featureGroupFeature.getName());
for (ValidationRule validationRule : expectation.getValidationRules()) {
if (validationRule.getFeatureType() != null && !getRuleTypeMappings(validationRule.getFeatureType(), featureGroupFeature.getType())) {
featureTypesErrMsg.add("feature: " + featureGroupFeature.getName() + ", " + "expectation: " + expectation.getExpectation().getName() + ", " + "feature type: " + featureGroupFeature.getType() + ", " + "rule: " + validationRule.getName() + ", " + "rule type: " + validationRule.getFeatureType() + ", ");
}
}
}
}
}
if (!featureTypesErrMsg.isEmpty()) {
throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURE_GROUP_EXPECTATION_FEATURE_TYPE_INVALID, FINE, featureTypesErrMsg.toString());
}
}
use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureStoreExpectation in project hopsworks by logicalclocks.
the class FeaturegroupController method persistFeaturegroupMetadata.
/**
* Persists metadata of a new feature group in the feature_group table
*
* @param featurestore the featurestore of the feature group
* @param user the Hopsworks user making the request
* @param featuregroupDTO DTO of the feature group
* @param cachedFeaturegroup the cached feature group that the feature group is linked to (if any)
* @param onDemandFeaturegroup the on-demand feature group that the feature group is linked to (if any)
* @return the created entity
*/
private Featuregroup persistFeaturegroupMetadata(Featurestore featurestore, Users user, FeaturegroupDTO featuregroupDTO, CachedFeaturegroup cachedFeaturegroup, OnDemandFeaturegroup onDemandFeaturegroup) throws FeaturestoreException {
Featuregroup featuregroup = new Featuregroup();
featuregroup.setName(featuregroupDTO.getName());
featuregroup.setFeaturestore(featurestore);
featuregroup.setCreated(new Date());
featuregroup.setCreator(user);
featuregroup.setVersion(featuregroupDTO.getVersion());
if (featuregroupDTO.getValidationType() != null) {
featuregroup.setValidationType(featuregroupDTO.getValidationType());
}
featuregroup.setFeaturegroupType(featuregroupDTO instanceof CachedFeaturegroupDTO ? FeaturegroupType.CACHED_FEATURE_GROUP : FeaturegroupType.ON_DEMAND_FEATURE_GROUP);
featuregroup.setCachedFeaturegroup(cachedFeaturegroup);
featuregroup.setOnDemandFeaturegroup(onDemandFeaturegroup);
featuregroup.setEventTime(featuregroupDTO.getEventTime());
StatisticsConfig statisticsConfig = new StatisticsConfig(featuregroupDTO.getStatisticsConfig().getEnabled(), featuregroupDTO.getStatisticsConfig().getCorrelations(), featuregroupDTO.getStatisticsConfig().getHistograms(), featuregroupDTO.getStatisticsConfig().getExactUniqueness());
statisticsConfig.setFeaturegroup(featuregroup);
statisticsConfig.setStatisticColumns(featuregroupDTO.getStatisticsConfig().getColumns().stream().map(sc -> new StatisticColumn(statisticsConfig, sc)).collect(Collectors.toList()));
featuregroup.setStatisticsConfig(statisticsConfig);
if (featuregroupDTO.getExpectationsNames() != null) {
List<FeatureGroupExpectation> featureGroupExpectations = new ArrayList<>();
for (String name : featuregroupDTO.getExpectationsNames()) {
FeatureStoreExpectation featureStoreExpectation = featureGroupValidationsController.getFeatureStoreExpectation(featuregroup.getFeaturestore(), name);
FeatureGroupExpectation featureGroupExpectation;
Optional<FeatureGroupExpectation> e = featureGroupExpectationFacade.findByFeaturegroupAndExpectation(featuregroup, featureStoreExpectation);
if (!e.isPresent()) {
featureGroupExpectation = new FeatureGroupExpectation();
featureGroupExpectation.setFeaturegroup(featuregroup);
featureGroupExpectation.setFeatureStoreExpectation(featureStoreExpectation);
} else {
featureGroupExpectation = e.get();
}
featureGroupExpectations.add(featureGroupExpectation);
}
featuregroup.setExpectations(featureGroupExpectations);
}
featuregroupFacade.persist(featuregroup);
return featuregroup;
}
use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureStoreExpectation in project hopsworks by logicalclocks.
the class FeatureStoreExpectationFacade method findByFeaturestore.
public CollectionInfo findByFeaturestore(Integer offset, Integer limit, Set<? extends FilterBy> filters, Set<? extends SortBy> sort, Featurestore featurestore) {
String queryStr = buildQuery("SELECT fse FROM FeatureStoreExpectation fse ", filters, sort, "fse.featureStore = :featureStore ");
String queryCountStr = buildQuery("SELECT COUNT(fse.id) FROM FeatureStoreExpectation fse ", filters, sort, "fse.featureStore = :featureStore ");
Query query = em.createQuery(queryStr, FeatureStoreExpectation.class).setParameter("featureStore", featurestore);
Query queryCount = em.createQuery(queryCountStr, FeatureStoreExpectation.class).setParameter("featureStore", featurestore);
setFilter(filters, query);
setFilter(filters, queryCount);
setOffsetAndLim(offset, limit, query);
return findAll(offset, limit, filters, query, queryCount);
}
Aggregations