Examples with Featuregroup io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup used on opensource projects

Example 26 with Featuregroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup in project hopsworks by logicalclocks.

the class FeaturegroupController method enableFeaturegroupOnline.

/**
 * Enable online feature serving of a feature group that is currently only offline
 *
 * @param featurestore    the featurestore where the featuregroup resides
 * @param featuregroupDTO the updated featuregroup metadata
 * @return DTO of the updated feature group
 * @throws FeaturestoreException
 */
public FeaturegroupDTO enableFeaturegroupOnline(Featurestore featurestore, FeaturegroupDTO featuregroupDTO, Project project, Users user) throws FeaturestoreException, SQLException, ServiceException, KafkaException, SchemaException, ProjectException, UserException, IOException, HopsSecurityException {
    Featuregroup featuregroup = getFeaturegroupById(featurestore, featuregroupDTO.getId());
    if (featuregroup.getFeaturegroupType() == FeaturegroupType.ON_DEMAND_FEATURE_GROUP) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ONLINE_FEATURE_SERVING_NOT_SUPPORTED_FOR_ON_DEMAND_FEATUREGROUPS, Level.FINE, ", Online feature serving is only supported for featuregroups of type: " + FeaturegroupType.CACHED_FEATURE_GROUP + ", and the user requested to enable feature serving on a " + "featuregroup with type:" + FeaturegroupType.ON_DEMAND_FEATURE_GROUP);
    }
    cachedFeaturegroupController.enableFeaturegroupOnline(featurestore, featuregroup, project, user);
    // Log activity
    fsActivityFacade.logMetadataActivity(user, featuregroup, FeaturestoreActivityMeta.ONLINE_ENABLED, null);
    return convertFeaturegrouptoDTO(featuregroup, project, user);
}

Example 27 with Featuregroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup in project hopsworks by logicalclocks.

the class FeaturegroupController method persistFeaturegroupMetadata.

/**
 * Persists metadata of a new feature group in the feature_group table
 *
 * @param featurestore the featurestore of the feature group
 * @param user the Hopsworks user making the request
 * @param featuregroupDTO DTO of the feature group
 * @param cachedFeaturegroup the cached feature group that the feature group is linked to (if any)
 * @param onDemandFeaturegroup the on-demand feature group that the feature group is linked to (if any)
 * @return the created entity
 */
private Featuregroup persistFeaturegroupMetadata(Featurestore featurestore, Users user, FeaturegroupDTO featuregroupDTO, CachedFeaturegroup cachedFeaturegroup, OnDemandFeaturegroup onDemandFeaturegroup) throws FeaturestoreException {
    Featuregroup featuregroup = new Featuregroup();
    featuregroup.setName(featuregroupDTO.getName());
    featuregroup.setFeaturestore(featurestore);
    featuregroup.setCreated(new Date());
    featuregroup.setCreator(user);
    featuregroup.setVersion(featuregroupDTO.getVersion());
    if (featuregroupDTO.getValidationType() != null) {
        featuregroup.setValidationType(featuregroupDTO.getValidationType());
    }
    featuregroup.setFeaturegroupType(featuregroupDTO instanceof CachedFeaturegroupDTO ? FeaturegroupType.CACHED_FEATURE_GROUP : FeaturegroupType.ON_DEMAND_FEATURE_GROUP);
    featuregroup.setCachedFeaturegroup(cachedFeaturegroup);
    featuregroup.setOnDemandFeaturegroup(onDemandFeaturegroup);
    featuregroup.setEventTime(featuregroupDTO.getEventTime());
    StatisticsConfig statisticsConfig = new StatisticsConfig(featuregroupDTO.getStatisticsConfig().getEnabled(), featuregroupDTO.getStatisticsConfig().getCorrelations(), featuregroupDTO.getStatisticsConfig().getHistograms(), featuregroupDTO.getStatisticsConfig().getExactUniqueness());
    statisticsConfig.setFeaturegroup(featuregroup);
    statisticsConfig.setStatisticColumns(featuregroupDTO.getStatisticsConfig().getColumns().stream().map(sc -> new StatisticColumn(statisticsConfig, sc)).collect(Collectors.toList()));
    featuregroup.setStatisticsConfig(statisticsConfig);
    if (featuregroupDTO.getExpectationsNames() != null) {
        List<FeatureGroupExpectation> featureGroupExpectations = new ArrayList<>();
        for (String name : featuregroupDTO.getExpectationsNames()) {
            FeatureStoreExpectation featureStoreExpectation = featureGroupValidationsController.getFeatureStoreExpectation(featuregroup.getFeaturestore(), name);
            FeatureGroupExpectation featureGroupExpectation;
            Optional<FeatureGroupExpectation> e = featureGroupExpectationFacade.findByFeaturegroupAndExpectation(featuregroup, featureStoreExpectation);
            if (!e.isPresent()) {
                featureGroupExpectation = new FeatureGroupExpectation();
                featureGroupExpectation.setFeaturegroup(featuregroup);
                featureGroupExpectation.setFeatureStoreExpectation(featureStoreExpectation);
            } else {
                featureGroupExpectation = e.get();
            }
            featureGroupExpectations.add(featureGroupExpectation);
        }
        featuregroup.setExpectations(featureGroupExpectations);
    }
    featuregroupFacade.persist(featuregroup);
    return featuregroup;
}

Example 28 with Featuregroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup in project hopsworks by logicalclocks.

the class PitJoinController method addEventTimeOn.

public List<Feature> addEventTimeOn(List<Feature> on, Featuregroup featureGroup, String fgAlias) {
    // make copy of features since otherwise it leads to problems when setting aliases later on
    List<Feature> newOn = on.stream().map(f -> new Feature(f.getName(), f.getFgAlias(), f.isPrimary())).collect(Collectors.toList());
    newOn.add(new Feature(featureGroup.getEventTime(), fgAlias));
    return newOn;
}

Example 29 with Featuregroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup in project hopsworks by logicalclocks.

the class QueryController method convertQueryDTO.

/**
 * Recursively convert the QueryDTO into the internal query representation
 * @param queryDTO
 * @return
 */
public Query convertQueryDTO(QueryDTO queryDTO, Map<Integer, String> fgAliasLookup, Map<Integer, Featuregroup> fgLookup, Map<Integer, List<Feature>> availableFeatureLookup, boolean pitEnabled) throws FeaturestoreException {
    Integer fgId = queryDTO.getLeftFeatureGroup().getId();
    Featuregroup fg = fgLookup.get(fgId);
    String featureStore = featurestoreFacade.getHiveDbName(fg.getFeaturestore().getHiveDbId());
    // used to build the online query - needs to respect the online db format name
    String projectName = onlineFeaturestoreController.getOnlineFeaturestoreDbName(fg.getFeaturestore().getProject());
    List<Feature> requestedFeatures = validateFeatures(fg, queryDTO.getLeftFeatures(), availableFeatureLookup.get(fgId));
    Query query = new Query(featureStore, projectName, fg, fgAliasLookup.get(fgId), requestedFeatures, availableFeatureLookup.get(fgId), queryDTO.getHiveEngine());
    if (fg.getCachedFeaturegroup() != null && fg.getCachedFeaturegroup().getTimeTravelFormat() == TimeTravelFormat.HUDI) {
        // if hudi and end hive engine, only possible to get latest snapshot else raise exception
        if (queryDTO.getHiveEngine() && (queryDTO.getLeftFeatureGroupEndTime() != null || queryDTO.getJoins().stream().anyMatch(join -> join.getQuery().getLeftFeatureGroupEndTime() != null))) {
            throw new IllegalArgumentException("Hive engine on Python environments does not support incremental or " + "snapshot queries. Read feature group without timestamp to retrieve latest snapshot or switch to " + "environment with Spark Engine.");
        }
        // If the feature group is hudi, validate and configure start and end commit id/timestamp
        FeatureGroupCommit endCommit = featureGroupCommitCommitController.findCommitByDate(fg, queryDTO.getLeftFeatureGroupEndTime());
        query.setLeftFeatureGroupEndTimestamp(endCommit.getCommittedOn());
        query.setLeftFeatureGroupEndCommitId(endCommit.getFeatureGroupCommitPK().getCommitId());
        if ((queryDTO.getJoins() == null || queryDTO.getJoins().isEmpty()) && queryDTO.getLeftFeatureGroupStartTime() != null) {
            Long exactStartCommitTimestamp = featureGroupCommitCommitController.findCommitByDate(query.getFeaturegroup(), queryDTO.getLeftFeatureGroupStartTime()).getCommittedOn();
            query.setLeftFeatureGroupStartTimestamp(exactStartCommitTimestamp);
        } else if (queryDTO.getJoins() != null && queryDTO.getLeftFeatureGroupStartTime() != null) {
            throw new IllegalArgumentException("For incremental queries start time must be provided and " + "join statements are not allowed");
        }
    }
    // If there are any joins, recursively convert the Join's QueryDTO into the internal Query representation
    if (queryDTO.getJoins() != null && !queryDTO.getJoins().isEmpty()) {
        query.setJoins(convertJoins(query, queryDTO.getJoins(), fgAliasLookup, fgLookup, availableFeatureLookup, pitEnabled));
        // remove duplicated join columns
        removeDuplicateColumns(query, pitEnabled);
    }
    // If there are any filters, recursively convert the
    if (queryDTO.getFilter() != null) {
        query.setFilter(filterController.convertFilterLogic(queryDTO.getFilter(), fgLookup, availableFeatureLookup));
    }
    return query;
}

Example 30 with Featuregroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup in project hopsworks by logicalclocks.

the class QueryController method populateFgLookupTables.

public int populateFgLookupTables(QueryDTO queryDTO, int fgId, Map<Integer, String> fgAliasLookup, Map<Integer, Featuregroup> fgLookup, Map<Integer, List<Feature>> availableFeatureLookup, Project project, Users user, String prefix) throws FeaturestoreException {
    // go into depth first
    if (queryDTO.getJoins() != null && !queryDTO.getJoins().isEmpty()) {
        for (JoinDTO join : queryDTO.getJoins()) {
            fgId = populateFgLookupTables(join.getQuery(), fgId, fgAliasLookup, fgLookup, availableFeatureLookup, project, user, join.getPrefix());
            fgId++;
        }
    }
    Featuregroup fg = validateFeaturegroupDTO(queryDTO.getLeftFeatureGroup());
    fgLookup.put(fg.getId(), fg);
    fgAliasLookup.put(fg.getId(), generateAs(fgId));
    List<Feature> availableFeatures = featuregroupController.getFeatures(fg, project, user).stream().map(f -> new Feature(f.getName(), fgAliasLookup.get(fg.getId()), f.getType(), f.getDefaultValue(), f.getPrimary(), fg, prefix)).collect(Collectors.toList());
    availableFeatureLookup.put(fg.getId(), availableFeatures);
    return fgId;
}