Search in sources :

Example 1 with FeatureGroupExpectation

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureGroupExpectation in project hopsworks by logicalclocks.

the class FeatureGroupValidationsController method attachExpectation.

public FeatureGroupExpectation attachExpectation(Featuregroup featuregroup, String name, Project project, Users user) throws FeaturestoreException {
    FeatureStoreExpectation featureStoreExpectation = getFeatureStoreExpectation(featuregroup.getFeaturestore(), name);
    FeatureGroupExpectation featureGroupExpectation;
    Optional<FeatureGroupExpectation> attachedExpectation = featureGroupExpectationFacade.findByFeaturegroupAndExpectation(featuregroup, featureStoreExpectation);
    featureValidation(featureStoreExpectation, featuregroup, project, user);
    if (!attachedExpectation.isPresent()) {
        featureGroupExpectation = new FeatureGroupExpectation();
        featureGroupExpectation.setFeaturegroup(featuregroup);
        featureGroupExpectation.setFeatureStoreExpectation(featureStoreExpectation);
        return featureGroupExpectationFacade.merge(featureGroupExpectation);
    } else {
        return attachedExpectation.get();
    }
}
Also used : FeatureGroupExpectation(io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureGroupExpectation) FeatureStoreExpectation(io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureStoreExpectation)

Example 2 with FeatureGroupExpectation

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureGroupExpectation in project hopsworks by logicalclocks.

the class FeaturegroupController method getFeaturegroupsForFeaturestore.

/**
 * Gets all featuregroups for a particular featurestore and project, using the userCerts to query Hive
 *
 * @param featurestore featurestore to query featuregroups for
 * @return list of XML/JSON DTOs of the featuregroups
 */
public List<FeaturegroupDTO> getFeaturegroupsForFeaturestore(Featurestore featurestore, Project project, Users user, Set<String> expectationNames) throws FeaturestoreException, ServiceException {
    List<Featuregroup> featuregroups = new ArrayList<>();
    if (expectationNames != null && !expectationNames.isEmpty()) {
        for (String name : expectationNames) {
            for (FeatureGroupExpectation featureGroupExpectation : featureStoreExpectationFacade.findByFeaturestoreAndName(featurestore, name).orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURE_STORE_EXPECTATION_NOT_FOUND, Level.FINE, name)).getFeatureGroupExpectations()) {
                if (featuregroups.isEmpty()) {
                    featuregroups.add(featureGroupExpectation.getFeaturegroup());
                } else {
                    boolean found = featuregroups.stream().anyMatch(fg -> fg.getId().equals(featureGroupExpectation.getFeaturegroup().getId()));
                    if (!found) {
                        featuregroups.add(featureGroupExpectation.getFeaturegroup());
                    }
                }
            }
        }
    } else {
        featuregroups = featuregroupFacade.findByFeaturestore(featurestore);
    }
    List<FeaturegroupDTO> featuregroupDTOS = new ArrayList<>();
    for (Featuregroup featuregroup : featuregroups) {
        featuregroupDTOS.add(convertFeaturegrouptoDTO(featuregroup, project, user));
    }
    return featuregroupDTOS;
}
Also used : FeatureGroupExpectation(io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureGroupExpectation) Featuregroup(io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup) OnDemandFeaturegroup(io.hops.hopsworks.persistence.entity.featurestore.featuregroup.ondemand.OnDemandFeaturegroup) CachedFeaturegroup(io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.CachedFeaturegroup) ArrayList(java.util.ArrayList) FeaturestoreException(io.hops.hopsworks.exceptions.FeaturestoreException) OnDemandFeaturegroupDTO(io.hops.hopsworks.common.featurestore.featuregroup.ondemand.OnDemandFeaturegroupDTO) CachedFeaturegroupDTO(io.hops.hopsworks.common.featurestore.featuregroup.cached.CachedFeaturegroupDTO)

Example 3 with FeatureGroupExpectation

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureGroupExpectation in project hopsworks by logicalclocks.

the class FeaturegroupController method persistFeaturegroupMetadata.

/**
 * Persists metadata of a new feature group in the feature_group table
 *
 * @param featurestore the featurestore of the feature group
 * @param user the Hopsworks user making the request
 * @param featuregroupDTO DTO of the feature group
 * @param cachedFeaturegroup the cached feature group that the feature group is linked to (if any)
 * @param onDemandFeaturegroup the on-demand feature group that the feature group is linked to (if any)
 * @return the created entity
 */
private Featuregroup persistFeaturegroupMetadata(Featurestore featurestore, Users user, FeaturegroupDTO featuregroupDTO, CachedFeaturegroup cachedFeaturegroup, OnDemandFeaturegroup onDemandFeaturegroup) throws FeaturestoreException {
    Featuregroup featuregroup = new Featuregroup();
    featuregroup.setName(featuregroupDTO.getName());
    featuregroup.setFeaturestore(featurestore);
    featuregroup.setCreated(new Date());
    featuregroup.setCreator(user);
    featuregroup.setVersion(featuregroupDTO.getVersion());
    if (featuregroupDTO.getValidationType() != null) {
        featuregroup.setValidationType(featuregroupDTO.getValidationType());
    }
    featuregroup.setFeaturegroupType(featuregroupDTO instanceof CachedFeaturegroupDTO ? FeaturegroupType.CACHED_FEATURE_GROUP : FeaturegroupType.ON_DEMAND_FEATURE_GROUP);
    featuregroup.setCachedFeaturegroup(cachedFeaturegroup);
    featuregroup.setOnDemandFeaturegroup(onDemandFeaturegroup);
    featuregroup.setEventTime(featuregroupDTO.getEventTime());
    StatisticsConfig statisticsConfig = new StatisticsConfig(featuregroupDTO.getStatisticsConfig().getEnabled(), featuregroupDTO.getStatisticsConfig().getCorrelations(), featuregroupDTO.getStatisticsConfig().getHistograms(), featuregroupDTO.getStatisticsConfig().getExactUniqueness());
    statisticsConfig.setFeaturegroup(featuregroup);
    statisticsConfig.setStatisticColumns(featuregroupDTO.getStatisticsConfig().getColumns().stream().map(sc -> new StatisticColumn(statisticsConfig, sc)).collect(Collectors.toList()));
    featuregroup.setStatisticsConfig(statisticsConfig);
    if (featuregroupDTO.getExpectationsNames() != null) {
        List<FeatureGroupExpectation> featureGroupExpectations = new ArrayList<>();
        for (String name : featuregroupDTO.getExpectationsNames()) {
            FeatureStoreExpectation featureStoreExpectation = featureGroupValidationsController.getFeatureStoreExpectation(featuregroup.getFeaturestore(), name);
            FeatureGroupExpectation featureGroupExpectation;
            Optional<FeatureGroupExpectation> e = featureGroupExpectationFacade.findByFeaturegroupAndExpectation(featuregroup, featureStoreExpectation);
            if (!e.isPresent()) {
                featureGroupExpectation = new FeatureGroupExpectation();
                featureGroupExpectation.setFeaturegroup(featuregroup);
                featureGroupExpectation.setFeatureStoreExpectation(featureStoreExpectation);
            } else {
                featureGroupExpectation = e.get();
            }
            featureGroupExpectations.add(featureGroupExpectation);
        }
        featuregroup.setExpectations(featureGroupExpectations);
    }
    featuregroupFacade.persist(featuregroup);
    return featuregroup;
}
Also used : StatisticsConfig(io.hops.hopsworks.persistence.entity.featurestore.statistics.StatisticsConfig) FeatureGroupExpectation(io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureGroupExpectation) Featuregroup(io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup) OnDemandFeaturegroup(io.hops.hopsworks.persistence.entity.featurestore.featuregroup.ondemand.OnDemandFeaturegroup) CachedFeaturegroup(io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.CachedFeaturegroup) ArrayList(java.util.ArrayList) CachedFeaturegroupDTO(io.hops.hopsworks.common.featurestore.featuregroup.cached.CachedFeaturegroupDTO) FeatureStoreExpectation(io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureStoreExpectation) StatisticColumn(io.hops.hopsworks.persistence.entity.featurestore.statistics.StatisticColumn) Date(java.util.Date)

Example 4 with FeatureGroupExpectation

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureGroupExpectation in project hopsworks by logicalclocks.

the class FeatureGroupExpectationFacade method findByFeaturegroup.

public CollectionInfo findByFeaturegroup(Integer offset, Integer limit, Set<? extends FilterBy> filter, Set<? extends SortBy> sort, Featuregroup featuregroup) {
    String queryStr = buildQuery("SELECT fge FROM FeatureGroupExpectation fge ", filter, sort, "fge.featuregroup = :featuregroup ");
    String queryCountStr = buildQuery("SELECT COUNT(fge.id) FROM FeatureGroupExpectation fge ", filter, sort, "fge.featuregroup = :featuregroup ");
    Query query = em.createQuery(queryStr, FeatureGroupExpectation.class).setParameter("featuregroup", featuregroup);
    Query queryCount = em.createQuery(queryCountStr, FeatureGroupExpectation.class).setParameter("featuregroup", featuregroup);
    return findAll(offset, limit, filter, query, queryCount);
}
Also used : Query(javax.persistence.Query) TypedQuery(javax.persistence.TypedQuery) FeatureGroupExpectation(io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureGroupExpectation)

Example 5 with FeatureGroupExpectation

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureGroupExpectation in project hopsworks by logicalclocks.

the class FeatureGroupValidationsController method getFeatureGroupExpectation.

public FeatureGroupExpectation getFeatureGroupExpectation(Featuregroup featuregroup, String name) throws FeaturestoreException {
    // Find the FeatureStoreExpectation first
    FeatureStoreExpectation featureStoreExpectation = getFeatureStoreExpectation(featuregroup.getFeaturestore(), name);
    Optional<FeatureGroupExpectation> optional = featureGroupExpectationFacade.findByFeaturegroupAndExpectation(featuregroup, featureStoreExpectation);
    if (!optional.isPresent()) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURE_GROUP_EXPECTATION_NOT_FOUND, FINE, "Expectation: " + name);
    }
    return optional.get();
}
Also used : FeatureGroupExpectation(io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureGroupExpectation) FeatureStoreExpectation(io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureStoreExpectation) FeaturestoreException(io.hops.hopsworks.exceptions.FeaturestoreException)

Aggregations

FeatureGroupExpectation (io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureGroupExpectation)9 FeatureStoreExpectation (io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.FeatureStoreExpectation)4 ExpectationDTO (io.hops.hopsworks.api.featurestore.datavalidation.expectations.ExpectationDTO)3 FeaturestoreException (io.hops.hopsworks.exceptions.FeaturestoreException)3 AllowedProjectRoles (io.hops.hopsworks.api.filter.AllowedProjectRoles)2 ApiKeyRequired (io.hops.hopsworks.api.filter.apiKey.ApiKeyRequired)2 ResourceRequest (io.hops.hopsworks.common.api.ResourceRequest)2 CachedFeaturegroupDTO (io.hops.hopsworks.common.featurestore.featuregroup.cached.CachedFeaturegroupDTO)2 JWTRequired (io.hops.hopsworks.jwt.annotation.JWTRequired)2 Featuregroup (io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup)2 CachedFeaturegroup (io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.CachedFeaturegroup)2 OnDemandFeaturegroup (io.hops.hopsworks.persistence.entity.featurestore.featuregroup.ondemand.OnDemandFeaturegroup)2 ApiOperation (io.swagger.annotations.ApiOperation)2 ArrayList (java.util.ArrayList)2 Path (javax.ws.rs.Path)2 Produces (javax.ws.rs.Produces)2 AbstractFacade (io.hops.hopsworks.common.dao.AbstractFacade)1 OnDemandFeaturegroupDTO (io.hops.hopsworks.common.featurestore.featuregroup.ondemand.OnDemandFeaturegroupDTO)1 StatisticColumn (io.hops.hopsworks.persistence.entity.featurestore.statistics.StatisticColumn)1 StatisticsConfig (io.hops.hopsworks.persistence.entity.featurestore.statistics.StatisticsConfig)1