use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureGroupExpectation in project hopsworks by logicalclocks.
the class FeatureGroupValidationsController method attachExpectation.
public FeatureGroupExpectation attachExpectation(Featuregroup featuregroup, String name, Project project, Users user) throws FeaturestoreException {
FeatureStoreExpectation featureStoreExpectation = getFeatureStoreExpectation(featuregroup.getFeaturestore(), name);
FeatureGroupExpectation featureGroupExpectation;
Optional<FeatureGroupExpectation> attachedExpectation = featureGroupExpectationFacade.findByFeaturegroupAndExpectation(featuregroup, featureStoreExpectation);
featureValidation(featureStoreExpectation, featuregroup, project, user);
if (!attachedExpectation.isPresent()) {
featureGroupExpectation = new FeatureGroupExpectation();
featureGroupExpectation.setFeaturegroup(featuregroup);
featureGroupExpectation.setFeatureStoreExpectation(featureStoreExpectation);
return featureGroupExpectationFacade.merge(featureGroupExpectation);
} else {
return attachedExpectation.get();
}
}
use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureGroupExpectation in project hopsworks by logicalclocks.
the class FeaturegroupController method getFeaturegroupsForFeaturestore.
/**
* Gets all featuregroups for a particular featurestore and project, using the userCerts to query Hive
*
* @param featurestore featurestore to query featuregroups for
* @return list of XML/JSON DTOs of the featuregroups
*/
public List<FeaturegroupDTO> getFeaturegroupsForFeaturestore(Featurestore featurestore, Project project, Users user, Set<String> expectationNames) throws FeaturestoreException, ServiceException {
List<Featuregroup> featuregroups = new ArrayList<>();
if (expectationNames != null && !expectationNames.isEmpty()) {
for (String name : expectationNames) {
for (FeatureGroupExpectation featureGroupExpectation : featureStoreExpectationFacade.findByFeaturestoreAndName(featurestore, name).orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURE_STORE_EXPECTATION_NOT_FOUND, Level.FINE, name)).getFeatureGroupExpectations()) {
if (featuregroups.isEmpty()) {
featuregroups.add(featureGroupExpectation.getFeaturegroup());
} else {
boolean found = featuregroups.stream().anyMatch(fg -> fg.getId().equals(featureGroupExpectation.getFeaturegroup().getId()));
if (!found) {
featuregroups.add(featureGroupExpectation.getFeaturegroup());
}
}
}
}
} else {
featuregroups = featuregroupFacade.findByFeaturestore(featurestore);
}
List<FeaturegroupDTO> featuregroupDTOS = new ArrayList<>();
for (Featuregroup featuregroup : featuregroups) {
featuregroupDTOS.add(convertFeaturegrouptoDTO(featuregroup, project, user));
}
return featuregroupDTOS;
}
use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureGroupExpectation in project hopsworks by logicalclocks.
the class FeaturegroupController method persistFeaturegroupMetadata.
/**
* Persists metadata of a new feature group in the feature_group table
*
* @param featurestore the featurestore of the feature group
* @param user the Hopsworks user making the request
* @param featuregroupDTO DTO of the feature group
* @param cachedFeaturegroup the cached feature group that the feature group is linked to (if any)
* @param onDemandFeaturegroup the on-demand feature group that the feature group is linked to (if any)
* @return the created entity
*/
private Featuregroup persistFeaturegroupMetadata(Featurestore featurestore, Users user, FeaturegroupDTO featuregroupDTO, CachedFeaturegroup cachedFeaturegroup, OnDemandFeaturegroup onDemandFeaturegroup) throws FeaturestoreException {
Featuregroup featuregroup = new Featuregroup();
featuregroup.setName(featuregroupDTO.getName());
featuregroup.setFeaturestore(featurestore);
featuregroup.setCreated(new Date());
featuregroup.setCreator(user);
featuregroup.setVersion(featuregroupDTO.getVersion());
if (featuregroupDTO.getValidationType() != null) {
featuregroup.setValidationType(featuregroupDTO.getValidationType());
}
featuregroup.setFeaturegroupType(featuregroupDTO instanceof CachedFeaturegroupDTO ? FeaturegroupType.CACHED_FEATURE_GROUP : FeaturegroupType.ON_DEMAND_FEATURE_GROUP);
featuregroup.setCachedFeaturegroup(cachedFeaturegroup);
featuregroup.setOnDemandFeaturegroup(onDemandFeaturegroup);
featuregroup.setEventTime(featuregroupDTO.getEventTime());
StatisticsConfig statisticsConfig = new StatisticsConfig(featuregroupDTO.getStatisticsConfig().getEnabled(), featuregroupDTO.getStatisticsConfig().getCorrelations(), featuregroupDTO.getStatisticsConfig().getHistograms(), featuregroupDTO.getStatisticsConfig().getExactUniqueness());
statisticsConfig.setFeaturegroup(featuregroup);
statisticsConfig.setStatisticColumns(featuregroupDTO.getStatisticsConfig().getColumns().stream().map(sc -> new StatisticColumn(statisticsConfig, sc)).collect(Collectors.toList()));
featuregroup.setStatisticsConfig(statisticsConfig);
if (featuregroupDTO.getExpectationsNames() != null) {
List<FeatureGroupExpectation> featureGroupExpectations = new ArrayList<>();
for (String name : featuregroupDTO.getExpectationsNames()) {
FeatureStoreExpectation featureStoreExpectation = featureGroupValidationsController.getFeatureStoreExpectation(featuregroup.getFeaturestore(), name);
FeatureGroupExpectation featureGroupExpectation;
Optional<FeatureGroupExpectation> e = featureGroupExpectationFacade.findByFeaturegroupAndExpectation(featuregroup, featureStoreExpectation);
if (!e.isPresent()) {
featureGroupExpectation = new FeatureGroupExpectation();
featureGroupExpectation.setFeaturegroup(featuregroup);
featureGroupExpectation.setFeatureStoreExpectation(featureStoreExpectation);
} else {
featureGroupExpectation = e.get();
}
featureGroupExpectations.add(featureGroupExpectation);
}
featuregroup.setExpectations(featureGroupExpectations);
}
featuregroupFacade.persist(featuregroup);
return featuregroup;
}
use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureGroupExpectation in project hopsworks by logicalclocks.
the class FeatureGroupExpectationFacade method findByFeaturegroup.
public CollectionInfo findByFeaturegroup(Integer offset, Integer limit, Set<? extends FilterBy> filter, Set<? extends SortBy> sort, Featuregroup featuregroup) {
String queryStr = buildQuery("SELECT fge FROM FeatureGroupExpectation fge ", filter, sort, "fge.featuregroup = :featuregroup ");
String queryCountStr = buildQuery("SELECT COUNT(fge.id) FROM FeatureGroupExpectation fge ", filter, sort, "fge.featuregroup = :featuregroup ");
Query query = em.createQuery(queryStr, FeatureGroupExpectation.class).setParameter("featuregroup", featuregroup);
Query queryCount = em.createQuery(queryCountStr, FeatureGroupExpectation.class).setParameter("featuregroup", featuregroup);
return findAll(offset, limit, filter, query, queryCount);
}
use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureGroupExpectation in project hopsworks by logicalclocks.
the class FeatureGroupValidationsController method getFeatureGroupExpectation.
public FeatureGroupExpectation getFeatureGroupExpectation(Featuregroup featuregroup, String name) throws FeaturestoreException {
// Find the FeatureStoreExpectation first
FeatureStoreExpectation featureStoreExpectation = getFeatureStoreExpectation(featuregroup.getFeaturestore(), name);
Optional<FeatureGroupExpectation> optional = featureGroupExpectationFacade.findByFeaturegroupAndExpectation(featuregroup, featureStoreExpectation);
if (!optional.isPresent()) {
throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURE_GROUP_EXPECTATION_NOT_FOUND, FINE, "Expectation: " + name);
}
return optional.get();
}
Aggregations