Examples with FilterLogic io.hops.hopsworks.common.featurestore.query.filter.FilterLogic used on opensource projects

Example 11 with FilterLogic

use of io.hops.hopsworks.common.featurestore.query.filter.FilterLogic in project hopsworks by logicalclocks.

the class TrainingDatasetControllerTest method testConvertToFilterLogic_featureValueCondition.

public void testConvertToFilterLogic_featureValueCondition() throws Exception {
    // fg.feature > fg.otherFeature
    List<TrainingDatasetFilter> filters = new ArrayList<>();
    filters.add(createTrainingDatasetFilter(createTrainingDatasetFilterCondition("test_f", GREATER_THAN, "test_f1", 0, 0), SINGLE, "L"));
    Map<String, Feature> featureLookup = Maps.newHashMap();
    Feature feature = new Feature("test_f", "fg0");
    Feature feature1 = new Feature("test_f1", "fg0");
    Featuregroup featuregroup = new Featuregroup();
    featuregroup.setId(0);
    feature.setFeatureGroup(featuregroup);
    feature1.setFeatureGroup(featuregroup);
    featureLookup.put("0.test_f", feature);
    featureLookup.put("0.test_f1", feature1);
    FilterLogic actual = target.convertToFilterLogic(filters, featureLookup, "L");
    Filter left = new Filter(feature, GREATER_THAN, new FilterValue(0, "fg0", "test_f1"));
    Assert.assertEquals(actual.getType(), SINGLE);
    Assert.assertEquals(actual.getLeftFilter(), left);
    Assert.assertNull(actual.getLeftLogic());
    Assert.assertNull(actual.getRightFilter());
    Assert.assertNull(actual.getRightLogic());
}

Example 12 with FilterLogic

use of io.hops.hopsworks.common.featurestore.query.filter.FilterLogic in project hopsworks by logicalclocks.

the class TrainingDatasetControllerTest method testConvertToFilterLogic_singleCondition.

public void testConvertToFilterLogic_singleCondition() throws Exception {
    // fg.feature > 1
    List<TrainingDatasetFilter> filters = new ArrayList<>();
    filters.add(createTrainingDatasetFilter(createTrainingDatasetFilterCondition("test_f", GREATER_THAN, "1", 0), SINGLE, "L"));
    Map<String, Feature> featureLookup = Maps.newHashMap();
    Feature feature = new Feature("test_f", "fg0");
    Featuregroup featuregroup = new Featuregroup();
    featuregroup.setId(0);
    feature.setFeatureGroup(featuregroup);
    featureLookup.put("0.test_f", feature);
    FilterLogic actual = target.convertToFilterLogic(filters, featureLookup, "L");
    Feature f1 = new Feature("test_f", "fg0", null, false, null, null);
    f1.setFeatureGroup(featuregroup);
    Filter left = new Filter(f1, GREATER_THAN, "1");
    Assert.assertEquals(actual.getType(), SINGLE);
    Assert.assertEquals(actual.getLeftFilter(), left);
    Assert.assertNull(actual.getLeftLogic());
    Assert.assertNull(actual.getRightFilter());
    Assert.assertNull(actual.getRightLogic());
}

Example 13 with FilterLogic

use of io.hops.hopsworks.common.featurestore.query.filter.FilterLogic in project hopsworks by logicalclocks.

the class TrainingDatasetControllerTest method testconvertToFilterEntities_leftLogicRightFilter.

@Test
public void testconvertToFilterEntities_leftLogicRightFilter() throws Exception {
    // fg.feature > 1 and fg.feature > 2
    // "fg.feature > 1" stores as filter logic and "fg.feature > 2" stores as filter
    TrainingDataset trainingDataset = new TrainingDataset();
    Feature f1 = new Feature("test_f", "fg0");
    FilterLogic head = new FilterLogic();
    head.setType(AND);
    FilterLogic left = new FilterLogic();
    Filter left_left = new Filter(f1, GREATER_THAN, "1");
    left.setLeftFilter(left_left);
    left.setType(SINGLE);
    head.setLeftLogic(left);
    Filter right = new Filter(f1, GREATER_THAN, "2");
    head.setRightFilter(right);
    List<TrainingDatasetFilter> actual = target.convertToFilterEntities(head, trainingDataset, "L");
    List<TrainingDatasetFilter> expected = new ArrayList<>();
    expected.add(createTrainingDatasetFilter(null, AND, "L"));
    expected.add(createTrainingDatasetFilter(createTrainingDatasetFilterCondition("test_f", GREATER_THAN, "2"), SINGLE, "L.R"));
    expected.add(createTrainingDatasetFilter(createTrainingDatasetFilterCondition("test_f", GREATER_THAN, "1"), SINGLE, "L.L"));
    Assert.assertEquals(expected.size(), actual.size());
    Assert.assertTrue(expected.containsAll(actual));
    Assert.assertTrue(actual.containsAll(expected));
}

Example 14 with FilterLogic

use of io.hops.hopsworks.common.featurestore.query.filter.FilterLogic in project hopsworks by logicalclocks.

the class TrainingDatasetControllerTest method testconvertToFilterEntities_featureComparison.

@Test
public void testconvertToFilterEntities_featureComparison() throws Exception {
    // fg.feature > fg.otherFeature and fg.feature > fg1.otherFeature
    TrainingDataset trainingDataset = new TrainingDataset();
    Feature f1 = new Feature("test_f", "fg0");
    FilterLogic head = new FilterLogic();
    head.setType(AND);
    FilterValue filterValueLeft = new FilterValue(0, "fg0", "test_f1");
    Filter left = new Filter(f1, GREATER_THAN, filterValueLeft);
    head.setLeftFilter(left);
    FilterValue filterValueRight = new FilterValue(1, "fg1", "test_f2");
    Filter right = new Filter(f1, GREATER_THAN, filterValueRight);
    head.setRightFilter(right);
    List<TrainingDatasetFilter> actual = target.convertToFilterEntities(head, trainingDataset, "L");
    List<TrainingDatasetFilter> expected = new ArrayList<>();
    expected.add(createTrainingDatasetFilter(null, AND, "L"));
    expected.add(createTrainingDatasetFilter(createTrainingDatasetFilterCondition("test_f", GREATER_THAN, "test_f1", null, 0), SINGLE, "L.L"));
    expected.add(createTrainingDatasetFilter(createTrainingDatasetFilterCondition("test_f", GREATER_THAN, "test_f2", null, 1), SINGLE, "L.R"));
    Assert.assertEquals(expected.size(), actual.size());
    Assert.assertTrue(expected.containsAll(actual));
    Assert.assertTrue(actual.containsAll(expected));
}

Example 15 with FilterLogic

use of io.hops.hopsworks.common.featurestore.query.filter.FilterLogic in project hopsworks by logicalclocks.

the class TrainingDatasetController method getQuery.

// TODO feature view: remove
/**
 * Reconstruct the query used to generate the training datset, fetching the features and the joins
 * in the proper order from the database.
 * @param trainingDataset
 * @return
 * @throws FeaturestoreException
 */
public Query getQuery(TrainingDataset trainingDataset, boolean withLabel, Project project, Users user, Boolean isHiveEngine) throws FeaturestoreException {
    if (!trainingDataset.isQuery()) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.TRAINING_DATASET_NO_QUERY, Level.FINE, "Inference vector is only available for datasets generated by queries");
    }
    List<TrainingDatasetJoin> joins = getJoinsSorted(trainingDataset);
    // Convert all the TrainingDatasetFeatures to QueryFeatures
    Map<Integer, String> fgAliasLookup = getAliasLookupTable(joins);
    // These features are for the select part and are from different feature groups
    // to respect the ordering, all selected features are added to the left most Query instead of splitting them
    // over the querys for their respective origin feature group
    List<TrainingDatasetFeature> tdFeatures = getFeaturesSorted(trainingDataset, withLabel);
    // Check that all the feature groups still exists, if not throw a reasonable error
    if (tdFeatures.stream().anyMatch(j -> j.getFeatureGroup() == null)) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.TRAINING_DATASET_QUERY_FG_DELETED, Level.FINE);
    }
    // Get available features for all involved feature groups once, and save in map fgId -> availableFeatures
    Map<Integer, List<Feature>> availableFeaturesLookup = new HashMap<>();
    for (TrainingDatasetJoin join : joins) {
        if (!availableFeaturesLookup.containsKey(join.getFeatureGroup().getId())) {
            List<Feature> availableFeatures = featuregroupController.getFeatures(join.getFeatureGroup(), project, user).stream().map(f -> new Feature(f.getName(), fgAliasLookup.get(join.getId()), f.getType(), f.getDefaultValue(), f.getPrimary(), join.getFeatureGroup(), join.getPrefix())).collect(Collectors.toList());
            availableFeaturesLookup.put(join.getFeatureGroup().getId(), availableFeatures);
        }
    }
    Map<String, Feature> featureLookup = availableFeaturesLookup.values().stream().flatMap(List::stream).collect(Collectors.toMap(f -> makeFeatureLookupKey(f.getFeatureGroup().getId(), f.getName()), f -> f, (f1, f2) -> f1));
    List<Feature> features = new ArrayList<>();
    for (TrainingDatasetFeature requestedFeature : tdFeatures) {
        Feature tdFeature = featureLookup.get(makeFeatureLookupKey(requestedFeature.getFeatureGroup().getId(), requestedFeature.getName()));
        if (tdFeature == null) {
            throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURE_DOES_NOT_EXIST, Level.FINE, "Feature: " + requestedFeature.getName() + " not found in feature group: " + requestedFeature.getFeatureGroup().getName());
        }
        // instantiate new feature since alias in available feature is not correct if fg is joined with itself
        Feature featureWithCorrectAlias = new Feature(tdFeature.getName(), fgAliasLookup.get(requestedFeature.getTrainingDatasetJoin().getId()), tdFeature.getType(), tdFeature.getDefaultValue(), tdFeature.getPrefix(), requestedFeature.getFeatureGroup(), requestedFeature.getIndex());
        features.add(featureWithCorrectAlias);
    }
    // Keep a map feature store id -> feature store name
    Map<Integer, String> fsLookup = getFsLookupTableJoins(joins);
    Query query = new Query(fsLookup.get(joins.get(0).getFeatureGroup().getFeaturestore().getId()), onlineFeaturestoreController.getOnlineFeaturestoreDbName(joins.get(0).getFeatureGroup().getFeaturestore().getProject()), joins.get(0).getFeatureGroup(), fgAliasLookup.get(joins.get(0).getId()), features, availableFeaturesLookup.get(joins.get(0).getFeatureGroup().getId()), isHiveEngine);
    // Set the remaining feature groups as join
    List<Join> queryJoins = new ArrayList<>();
    for (int i = 1; i < joins.size(); i++) {
        // left side of the join stays fixed, the counter starts at 1
        queryJoins.add(getQueryJoin(query, joins.get(i), fgAliasLookup, fsLookup, availableFeaturesLookup, isHiveEngine));
    }
    query.setJoins(queryJoins);
    FilterLogic filterLogic = convertToFilterLogic(trainingDataset.getFilters(), featureLookup, "L");
    query.setFilter(filterLogic);
    return query;
}