Examples with FeaturegroupDTO io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupDTO used on opensource projects

Example 6 with FeaturegroupDTO

use of io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupDTO in project hopsworks by logicalclocks.

the class TestFeatureStoreInputValidation method testVerifyUserInputFeatureGroup.

@Test
public void testVerifyUserInputFeatureGroup() throws Exception {
    FeaturegroupDTO featuregroupDTO = new FeaturegroupDTO(1, "featurestore", 1, "1wrong_name", 1, "online_topic_name");
    // upper case
    featuregroupDTO.setName("UPPER_CASE");
    thrown.expect(FeaturestoreException.class);
    featurestoreInputValidation.verifyUserInput(featuregroupDTO);
    // starts with numeric
    featuregroupDTO.setName("12_numeric");
    thrown.expect(FeaturestoreException.class);
    featurestoreInputValidation.verifyUserInput(featuregroupDTO);
    // too long
    featuregroupDTO.setName(StringUtils.repeat("a", 260));
    thrown.expect(FeaturestoreException.class);
    featurestoreInputValidation.verifyUserInput(featuregroupDTO);
    // empty
    featuregroupDTO.setName("");
    thrown.expect(FeaturestoreException.class);
    featurestoreInputValidation.verifyUserInput(featuregroupDTO);
}

Example 7 with FeaturegroupDTO

use of io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupDTO in project hopsworks by logicalclocks.

the class TestPitJoinController method testIsPitEnabledFalseQueryDTO.

@Test
public void testIsPitEnabledFalseQueryDTO() {
    FeaturegroupDTO fg1 = new FeaturegroupDTO("ts");
    // not event time enabled fg, therefore should be pitEnabled should be false
    FeaturegroupDTO fg2 = new FeaturegroupDTO();
    JoinDTO join1 = new JoinDTO(new QueryDTO(fg1, null), null, null);
    JoinDTO join2 = new JoinDTO(new QueryDTO(fg2, null), null, null);
    List<JoinDTO> joins = Arrays.asList(join1, join2);
    QueryDTO leftQuery = new QueryDTO(new FeaturegroupDTO("ts"), new ArrayList<>(), joins);
    Assert.assertEquals(false, pitJoinController.isPitEnabled(leftQuery));
}

Example 8 with FeaturegroupDTO

use of io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupDTO in project hopsworks by logicalclocks.

the class TestStatisticColumnController method testVerifyStatisticColumnsExistFg.

@Test
public void testVerifyStatisticColumnsExistFg() throws Exception {
    FeaturegroupDTO featuregroupDTO = new FeaturegroupDTO();
    featuregroupDTO.setName("fg1");
    featuregroupDTO.setVersion(1);
    StatisticsConfigDTO statisticsConfig = new StatisticsConfigDTO();
    featuregroupDTO.setStatisticsConfig(statisticsConfig);
    statisticsConfig.setColumns(Arrays.asList("ft1", "ft4"));
    featuregroupDTO.setFeatures(Arrays.asList(new FeatureGroupFeatureDTO("ft1", null, ""), new FeatureGroupFeatureDTO("ft2", null, ""), new FeatureGroupFeatureDTO("ft3", null, "")));
    // should throw exception
    thrown.expect(FeaturestoreException.class);
    statisticColumnController.verifyStatisticColumnsExist(featuregroupDTO);
    // should not throw exception
    statisticsConfig.setColumns(Arrays.asList("ft1", "ft2"));
    statisticColumnController.verifyStatisticColumnsExist(featuregroupDTO);
}

Example 9 with FeaturegroupDTO

use of io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupDTO in project hopsworks by logicalclocks.

the class TestConstructorController method testHandleJoiningKeyRightSide.

@Test
public void testHandleJoiningKeyRightSide() throws Exception {
    // When specifying the "on" condition we remove the duplicates for the joining condition
    // there was a bug for which if the joining feature was selected only on the right side of the join,
    // it was removed. The joining feature should be removed only if it's present on both side,
    // i.e. it's a duplicate. This test make sure we don't regress.
    Mockito.when(featuregroupController.getFeatures(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(fg1FeaturesDTO, fg2FeaturesDTO);
    Mockito.when(featuregroupFacade.findById(Mockito.any())).thenReturn(Optional.of(fg1), Optional.of(fg2));
    Mockito.when(featurestoreFacade.getHiveDbName(Mockito.any())).thenReturn("fg1", "fg2");
    FeaturegroupDTO fg1 = new FeaturegroupDTO();
    fg1.setId(1);
    FeaturegroupDTO fg2 = new FeaturegroupDTO();
    fg2.setId(2);
    List<FeatureGroupFeatureDTO> leftRequestedFeatures = new ArrayList<>();
    leftRequestedFeatures.add(new FeatureGroupFeatureDTO("fg1_ft2"));
    List<FeatureGroupFeatureDTO> rightRequestedFeatures = new ArrayList<>();
    rightRequestedFeatures.addAll(Arrays.asList(new FeatureGroupFeatureDTO("fg2_ft2"), new FeatureGroupFeatureDTO("pr")));
    QueryDTO rightQueryDTO = new QueryDTO(fg2, rightRequestedFeatures);
    JoinDTO joinDTO = new JoinDTO(rightQueryDTO, null, null);
    QueryDTO queryDTO = new QueryDTO(fg1, leftRequestedFeatures, Arrays.asList(joinDTO));
    Map<Integer, String> fgAliasLookup = new HashMap<>();
    Map<Integer, Featuregroup> fgLookup = new HashMap<>();
    Map<Integer, List<Feature>> availableFeatureLookup = new HashMap<>();
    queryController.populateFgLookupTables(queryDTO, 1, fgAliasLookup, fgLookup, availableFeatureLookup, project, user, null);
    Query query = queryController.convertQueryDTO(queryDTO, fgAliasLookup, fgLookup, availableFeatureLookup, false);
    List<Feature> extractedFeatures = target.collectFeatures(query);
    // Make sure both features have been returned.
    // It's going to be 3 as the feature "pr" will be identified as primary key and joining key
    // so it's not going to be duplicated
    Assert.assertEquals(3, extractedFeatures.size());
    // Make sure the method sets the feature group name
    Assert.assertTrue(extractedFeatures.get(0).getFgAlias(false).equals("fg1") || extractedFeatures.get(0).getFgAlias(false).equals("fg2"));
}

Example 10 with FeaturegroupDTO

use of io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupDTO in project hopsworks by logicalclocks.

the class TestConstructorController method testExtractFeaturesBothSides.

@Test
public void testExtractFeaturesBothSides() throws Exception {
    Mockito.when(featuregroupController.getFeatures(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(fg1FeaturesDTO, fg2FeaturesDTO);
    Mockito.when(featuregroupFacade.findById(Mockito.any())).thenReturn(Optional.of(fg1), Optional.of(fg2));
    Mockito.when(featurestoreFacade.getHiveDbName(Mockito.any())).thenReturn("fg1", "fg2");
    FeaturegroupDTO fg1 = new FeaturegroupDTO();
    fg1.setId(1);
    FeaturegroupDTO fg2 = new FeaturegroupDTO();
    fg2.setId(2);
    List<FeatureGroupFeatureDTO> requestedFeatures = new ArrayList<>();
    requestedFeatures.add(new FeatureGroupFeatureDTO("*"));
    QueryDTO rightQueryDTO = new QueryDTO(fg2, requestedFeatures);
    JoinDTO joinDTO = new JoinDTO(rightQueryDTO, null, null);
    QueryDTO queryDTO = new QueryDTO(fg1, requestedFeatures, Arrays.asList(joinDTO));
    Map<Integer, String> fgAliasLookup = new HashMap<>();
    Map<Integer, Featuregroup> fgLookup = new HashMap<>();
    Map<Integer, List<Feature>> availableFeatureLookup = new HashMap<>();
    queryController.populateFgLookupTables(queryDTO, 1, fgAliasLookup, fgLookup, availableFeatureLookup, project, user, null);
    Query query = queryController.convertQueryDTO(queryDTO, fgAliasLookup, fgLookup, availableFeatureLookup, false);
    List<Feature> extractedFeatures = target.collectFeatures(query);
    // Make sure both features have been returned.
    // It's going to be 3 as the feature "pr" will be identified as primary key and joining key
    // so it's not going to be duplicated
    Assert.assertEquals(3, extractedFeatures.size());
    // Make sure the method sets the feature group name
    Assert.assertTrue(extractedFeatures.get(0).getFgAlias(false).equals("fg1") || extractedFeatures.get(0).getFgAlias(false).equals("fg2"));
}