Examples with CachedFeaturegroupDTO io.hops.hopsworks.common.featurestore.featuregroup.cached.CachedFeaturegroupDTO used on opensource projects

Example 6 with CachedFeaturegroupDTO

use of io.hops.hopsworks.common.featurestore.featuregroup.cached.CachedFeaturegroupDTO in project hopsworks by logicalclocks.

the class FeaturegroupController method persistFeaturegroupMetadata.

/**
 * Persists metadata of a new feature group in the feature_group table
 *
 * @param featurestore the featurestore of the feature group
 * @param user the Hopsworks user making the request
 * @param featuregroupDTO DTO of the feature group
 * @param cachedFeaturegroup the cached feature group that the feature group is linked to (if any)
 * @param onDemandFeaturegroup the on-demand feature group that the feature group is linked to (if any)
 * @return the created entity
 */
private Featuregroup persistFeaturegroupMetadata(Featurestore featurestore, Users user, FeaturegroupDTO featuregroupDTO, CachedFeaturegroup cachedFeaturegroup, OnDemandFeaturegroup onDemandFeaturegroup) throws FeaturestoreException {
    Featuregroup featuregroup = new Featuregroup();
    featuregroup.setName(featuregroupDTO.getName());
    featuregroup.setFeaturestore(featurestore);
    featuregroup.setCreated(new Date());
    featuregroup.setCreator(user);
    featuregroup.setVersion(featuregroupDTO.getVersion());
    if (featuregroupDTO.getValidationType() != null) {
        featuregroup.setValidationType(featuregroupDTO.getValidationType());
    }
    featuregroup.setFeaturegroupType(featuregroupDTO instanceof CachedFeaturegroupDTO ? FeaturegroupType.CACHED_FEATURE_GROUP : FeaturegroupType.ON_DEMAND_FEATURE_GROUP);
    featuregroup.setCachedFeaturegroup(cachedFeaturegroup);
    featuregroup.setOnDemandFeaturegroup(onDemandFeaturegroup);
    featuregroup.setEventTime(featuregroupDTO.getEventTime());
    StatisticsConfig statisticsConfig = new StatisticsConfig(featuregroupDTO.getStatisticsConfig().getEnabled(), featuregroupDTO.getStatisticsConfig().getCorrelations(), featuregroupDTO.getStatisticsConfig().getHistograms(), featuregroupDTO.getStatisticsConfig().getExactUniqueness());
    statisticsConfig.setFeaturegroup(featuregroup);
    statisticsConfig.setStatisticColumns(featuregroupDTO.getStatisticsConfig().getColumns().stream().map(sc -> new StatisticColumn(statisticsConfig, sc)).collect(Collectors.toList()));
    featuregroup.setStatisticsConfig(statisticsConfig);
    if (featuregroupDTO.getExpectationsNames() != null) {
        List<FeatureGroupExpectation> featureGroupExpectations = new ArrayList<>();
        for (String name : featuregroupDTO.getExpectationsNames()) {
            FeatureStoreExpectation featureStoreExpectation = featureGroupValidationsController.getFeatureStoreExpectation(featuregroup.getFeaturestore(), name);
            FeatureGroupExpectation featureGroupExpectation;
            Optional<FeatureGroupExpectation> e = featureGroupExpectationFacade.findByFeaturegroupAndExpectation(featuregroup, featureStoreExpectation);
            if (!e.isPresent()) {
                featureGroupExpectation = new FeatureGroupExpectation();
                featureGroupExpectation.setFeaturegroup(featuregroup);
                featureGroupExpectation.setFeatureStoreExpectation(featureStoreExpectation);
            } else {
                featureGroupExpectation = e.get();
            }
            featureGroupExpectations.add(featureGroupExpectation);
        }
        featuregroup.setExpectations(featureGroupExpectations);
    }
    featuregroupFacade.persist(featuregroup);
    return featuregroup;
}

Example 7 with CachedFeaturegroupDTO

use of io.hops.hopsworks.common.featurestore.featuregroup.cached.CachedFeaturegroupDTO in project hopsworks by logicalclocks.

the class ConstructorController method getHudiAliases.

public List<HudiFeatureGroupAliasDTO> getHudiAliases(Query query, List<HudiFeatureGroupAliasDTO> aliases, Project project, Users user) throws FeaturestoreException, ServiceException {
    if (query.getFeaturegroup().getFeaturegroupType() == FeaturegroupType.CACHED_FEATURE_GROUP && query.getFeaturegroup().getCachedFeaturegroup().getTimeTravelFormat() == TimeTravelFormat.HUDI) {
        CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(query.getFeaturegroup());
        Featuregroup featuregroup = query.getFeaturegroup();
        List<FeatureGroupFeatureDTO> featureGroupFeatureDTOS = cachedFeaturegroupController.getFeaturesDTO(featuregroup, project, user);
        featuregroupDTO.setFeatures(featureGroupFeatureDTOS);
        featuregroupDTO.setLocation(featurestoreUtils.resolveLocationURI(featuregroup.getCachedFeaturegroup().getHiveTbls().getSdId().getLocation()));
        if (query.getLeftFeatureGroupStartTimestamp() == null) {
            aliases.add(new HudiFeatureGroupAliasDTO(query.getAs(), featuregroupDTO, query.getLeftFeatureGroupEndTimestamp()));
        } else {
            aliases.add(new HudiFeatureGroupAliasDTO(query.getAs(), featuregroupDTO, query.getLeftFeatureGroupStartTimestamp(), query.getLeftFeatureGroupEndTimestamp()));
        }
    }
    if (query.getJoins() != null && !query.getJoins().isEmpty()) {
        for (Join join : query.getJoins()) {
            getHudiAliases(join.getRightQuery(), aliases, project, user);
        }
    }
    return aliases;
}