Examples with Featuregroup io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup used on opensource projects

Example 21 with Featuregroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup in project hopsworks by logicalclocks.

the class TestFeatureGroupCommitController method setup.

@Before
public void setup() {
    Inode inode = new Inode();
    HiveSds hiveSds = new HiveSds();
    hiveSds.setSdId(1l);
    hiveSds.setLocation("hopsfs://namenode.service.consul:8020/apps/hive/warehouse/test_proj_featurestore.db/fg1_1");
    hiveSds.setInode(inode);
    HiveTbls hiveTbls = new HiveTbls();
    hiveTbls.setSdId(hiveSds);
    hiveTbls.setTblName("fg1_1");
    CachedFeaturegroup cachedFeaturegroup = new CachedFeaturegroup();
    cachedFeaturegroup.setHiveTbls(hiveTbls);
    fs = new Featurestore();
    fs.setHiveDbId(1l);
    fs.setProject(new Project("test_proj"));
    fg1 = new Featuregroup(1);
    fg1.setName("fg1_1");
    fg1.setVersion(1);
    fg1.setFeaturestore(fs);
    fg1.setCachedFeaturegroup(cachedFeaturegroup);
}

Example 22 with Featuregroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup in project hopsworks by logicalclocks.

the class TestConstructorController method setup.

@Before
public void setup() {
    fs = new Featurestore();
    fs.setHiveDbId(1l);
    fs.setProject(new Project("test_proj"));
    cachedFeaturegroup = new CachedFeaturegroup();
    cachedFeaturegroup.setTimeTravelFormat(TimeTravelFormat.NONE);
    fg1 = new Featuregroup(1);
    fg1.setName("fg1");
    fg1.setVersion(1);
    fg1.setCachedFeaturegroup(cachedFeaturegroup);
    fg1.setFeaturestore(fs);
    fg2 = new Featuregroup(2);
    fg2.setName("fg2");
    fg2.setVersion(1);
    fg2.setCachedFeaturegroup(cachedFeaturegroup);
    fg2.setFeaturestore(fs);
    fg3 = new Featuregroup(3);
    fg3.setName("fg3");
    fg3.setVersion(1);
    fg3.setCachedFeaturegroup(cachedFeaturegroup);
    fg3.setFeaturestore(fs);
    fg4 = new Featuregroup(4);
    fg4.setName("fg4");
    fg4.setVersion(1);
    fg4.setCachedFeaturegroup(cachedFeaturegroup);
    fg4.setFeaturestore(fs);
    fgHudi = new Featuregroup(5);
    fgHudi.setName("fgHudi");
    fgHudi.setVersion(1);
    hudiFeatureGroup = new CachedFeaturegroup();
    hudiFeatureGroup.setTimeTravelFormat(TimeTravelFormat.HUDI);
    fgHudi.setCachedFeaturegroup(hudiFeatureGroup);
    fgHudi.setFeaturestore(fs);
    fg1Features = new ArrayList<>();
    fg1Features.add(new Feature("pr", "", true));
    fg1Features.add(new Feature("fg1_ft2", "", false));
    fg1FeaturesDTO = new ArrayList<>();
    fg1FeaturesDTO.add(new FeatureGroupFeatureDTO("pr", "Integer", "", true, false, "", null));
    fg1FeaturesDTO.add(new FeatureGroupFeatureDTO("fg1_ft2", "String", "", false, false, "", null));
    fg2Features = new ArrayList<>();
    fg2Features.add(new Feature("pr", "", true));
    fg2Features.add(new Feature("fg2_ft2", "", false));
    fg2FeaturesDTO = new ArrayList<>();
    fg2FeaturesDTO.add(new FeatureGroupFeatureDTO("pr", "Integer", "", true, false, "", null));
    fg2FeaturesDTO.add(new FeatureGroupFeatureDTO("fg2_ft2", "String", "", false, false, "", null));
    fg3Features = new ArrayList<>();
    fg3Features.add(new Feature("fg3_ft1", "", true));
    fg3Features.add(new Feature("fg3_ft2", "", false));
    fg4Features = new ArrayList<>();
    fg4Features.add(new Feature("pr", "fg4", true));
    fg4Features.add(new Feature("fg4_ft4_1", "fg4", "Float", null, "prefix4_"));
    fg4Features.add(new Feature("fg4_ft4_2", "fg4", "Float", null, "prefix4_"));
    fg4Features.add(new Feature("_hoodie_record_key", "fg4", "String", null, null));
    fg4Features.add(new Feature("_hoodie_partition_path", "fg4", "String", null, null));
    fg4Features.add(new Feature("_hoodie_commit_time", "fg4", "String", null, null));
    fg4Features.add(new Feature("_hoodie_file_name", "fg4", "String", null, null));
    fg4Features.add(new Feature("_hoodie_commit_seqno", "fg4", "String", null, null));
    singleEqualsJoinOperator = Arrays.asList(SqlCondition.EQUALS);
    featuregroupController = Mockito.mock(FeaturegroupController.class);
    featuregroupFacade = Mockito.mock(FeaturegroupFacade.class);
    featurestoreFacade = Mockito.mock(FeaturestoreFacade.class);
    onlineFeaturestoreController = Mockito.mock(OnlineFeaturestoreController.class);
    cachedFeaturegroupController = Mockito.mock(CachedFeaturegroupController.class);
    project = Mockito.mock(Project.class);
    user = Mockito.mock(Users.class);
    filterController = new FilterController(new ConstructorController());
    target = new ConstructorController(featuregroupController, featurestoreFacade, featuregroupFacade, onlineFeaturestoreController, cachedFeaturegroupController, filterController, new JoinController(new ConstructorController()));
    new JoinController(new ConstructorController());
    queryController = new QueryController(featuregroupController, featuregroupFacade, filterController, featurestoreFacade, onlineFeaturestoreController);
}

Example 23 with Featuregroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup in project hopsworks by logicalclocks.

the class FeaturegroupController method getFeaturegroupWithNameAndFeaturestore.

/**
 * Retrieves a list of feature groups with a specific name from a specific feature store
 *
 * @param name name of the featuregroup
 * @param featurestore the featurestore that the featuregroup belongs to
 * @return XML/JSON DTO of the featuregroup
 */
public List<FeaturegroupDTO> getFeaturegroupWithNameAndFeaturestore(Featurestore featurestore, String name, Project project, Users user) throws FeaturestoreException, ServiceException {
    List<Featuregroup> featuregroups = verifyFeaturegroupName(featurestore, name);
    List<FeaturegroupDTO> featuregroupDTOS = new ArrayList<>();
    for (Featuregroup featuregroup : featuregroups) {
        featuregroupDTOS.add(convertFeaturegrouptoDTO(featuregroup, project, user));
    }
    return featuregroupDTOS;
}

Example 24 with Featuregroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup in project hopsworks by logicalclocks.

the class FeaturegroupController method updateFeatureGroupStatsConfig.

/**
 * Updates statistics settings for a featuregroup
 *
 * @param featurestore    the featurestore where the featuregroup resides
 * @param featureGroupDTO a DTO containing the updated featuregroup stats
 * @return DTO of the updated feature group
 * @throws FeaturestoreException
 */
public FeaturegroupDTO updateFeatureGroupStatsConfig(Featurestore featurestore, FeaturegroupDTO featureGroupDTO, Project project, Users user) throws FeaturestoreException, ServiceException {
    Featuregroup featuregroup = getFeaturegroupById(featurestore, featureGroupDTO.getId());
    if (featureGroupDTO.getStatisticsConfig().getEnabled() != null) {
        featuregroup.getStatisticsConfig().setDescriptive(featureGroupDTO.getStatisticsConfig().getEnabled());
    }
    if (featureGroupDTO.getStatisticsConfig().getHistograms() != null) {
        featuregroup.getStatisticsConfig().setHistograms(featureGroupDTO.getStatisticsConfig().getHistograms());
    }
    if (featureGroupDTO.getStatisticsConfig().getCorrelations() != null) {
        featuregroup.getStatisticsConfig().setCorrelations(featureGroupDTO.getStatisticsConfig().getCorrelations());
    }
    if (featureGroupDTO.getStatisticsConfig().getExactUniqueness() != null) {
        featuregroup.getStatisticsConfig().setExactUniqueness(featureGroupDTO.getStatisticsConfig().getExactUniqueness());
    }
    // compare against schema from database, as client doesn't need to send schema in update request
    statisticColumnController.verifyStatisticColumnsExist(featureGroupDTO, featuregroup, getFeatures(featuregroup, project, user));
    featuregroupFacade.updateFeaturegroupMetadata(featuregroup);
    statisticColumnController.persistStatisticColumns(featuregroup, featureGroupDTO.getStatisticsConfig().getColumns());
    // get feature group again with persisted columns - this trip to the database can be saved
    featuregroup = getFeaturegroupById(featurestore, featureGroupDTO.getId());
    return convertFeaturegrouptoDTO(featuregroup, project, user);
}

Example 25 with Featuregroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup in project hopsworks by logicalclocks.

the class FeaturegroupController method updateValidationType.

/**
 * Updated validation type for a featuregroup
 *
 * @param featuregroup    the feature group to update
 * @return DTO of the updated feature group
 * @throws FeaturestoreException
 */
public FeaturegroupDTO updateValidationType(Featuregroup featuregroup, ValidationType validationType, Project project, Users user) throws FeaturestoreException, ServiceException {
    Featuregroup toUpdate = featuregroupFacade.findByNameVersionAndFeaturestore(featuregroup.getName(), featuregroup.getVersion(), featuregroup.getFeaturestore()).orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATUREGROUP_NOT_FOUND, Level.FINE, "featuregroup: " + featuregroup.getName()));
    toUpdate.setValidationType(validationType);
    featuregroupFacade.updateFeaturegroupMetadata(toUpdate);
    return convertFeaturegrouptoDTO(toUpdate, project, user);
}