Examples with FeatureView io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView used on opensource projects

Example 11 with FeatureView

use of io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView in project hopsworks by logicalclocks.

the class FeatureViewController method makeQuery.

public Query makeQuery(FeatureView featureView, Project project, Users user) throws FeaturestoreException {
    List<TrainingDatasetJoin> joins = featureView.getJoins().stream().sorted(Comparator.comparing(TrainingDatasetJoin::getIndex)).collect(Collectors.toList());
    Map<Integer, String> fgAliasLookup = trainingDatasetController.getAliasLookupTable(joins);
    List<TrainingDatasetFeature> tdFeatures = featureView.getFeatures().stream().sorted((t1, t2) -> {
        if (t1.getIndex() != null) {
            // compare based on index
            return t1.getIndex().compareTo(t2.getIndex());
        } else {
            // Old training dataset with no index. compare based on name
            return t1.getName().compareTo(t2.getName());
        }
    }).collect(Collectors.toList());
    // Check that all the feature groups still exists, if not throw a reasonable error
    if (tdFeatures.stream().anyMatch(j -> j.getFeatureGroup() == null)) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.TRAINING_DATASET_QUERY_FG_DELETED, Level.FINE);
    }
    // Get available features for all involved feature groups once, and save in map fgId -> availableFeatures
    Map<Integer, List<Feature>> availableFeaturesLookup = new HashMap<>();
    for (TrainingDatasetJoin join : joins) {
        if (!availableFeaturesLookup.containsKey(join.getFeatureGroup().getId())) {
            List<Feature> availableFeatures = featuregroupController.getFeatures(join.getFeatureGroup(), project, user).stream().map(f -> new Feature(f.getName(), fgAliasLookup.get(join.getId()), f.getType(), f.getPrimary(), f.getDefaultValue(), join.getPrefix())).collect(Collectors.toList());
            availableFeaturesLookup.put(join.getFeatureGroup().getId(), availableFeatures);
        }
    }
    List<Feature> features = new ArrayList<>();
    for (TrainingDatasetFeature requestedFeature : tdFeatures) {
        features.add(availableFeaturesLookup.get(requestedFeature.getFeatureGroup().getId()).stream().filter(af -> af.getName().equals(requestedFeature.getName())).map(af -> new Feature(af.getName(), fgAliasLookup.get(requestedFeature.getTrainingDatasetJoin().getId()), af.getType(), af.getDefaultValue(), af.getPrefix(), requestedFeature.getFeatureGroup(), requestedFeature.getIndex())).findFirst().orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURE_DOES_NOT_EXIST, Level.FINE, "Feature: " + requestedFeature.getName() + " not found in feature group: " + requestedFeature.getFeatureGroup().getName())));
    }
    // Keep a map feature store id -> feature store name
    Map<Integer, String> fsLookup = trainingDatasetController.getFsLookupTableJoins(joins);
    Query query = new Query(fsLookup.get(joins.get(0).getFeatureGroup().getFeaturestore().getId()), onlineFeaturestoreController.getOnlineFeaturestoreDbName(joins.get(0).getFeatureGroup().getFeaturestore().getProject()), joins.get(0).getFeatureGroup(), fgAliasLookup.get(joins.get(0).getId()), features, availableFeaturesLookup.get(joins.get(0).getFeatureGroup().getId()), false);
    // Set the remaining feature groups as join
    List<Join> queryJoins = new ArrayList<>();
    for (int i = 1; i < joins.size(); i++) {
        // left side of the join stays fixed, the counter starts at 1
        queryJoins.add(trainingDatasetController.getQueryJoin(query, joins.get(i), fgAliasLookup, fsLookup, availableFeaturesLookup, false));
    }
    query.setJoins(queryJoins);
    return query;
}