Examples with CachedFeaturegroup io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.CachedFeaturegroup used on opensource projects

Example 1 with CachedFeaturegroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.CachedFeaturegroup in project hopsworks by logicalclocks.

the class CachedFeaturegroupController method enableFeaturegroupOnline.

/**
 * Update a cached featuregroup that currently does not support online feature serving, to support it.
 *
 * @param featurestore the featurestore where the featuregroup resides
 * @param featuregroup the featuregroup entity to update
 * @param user the user making the request
 * @return a DTO of the updated featuregroup
 * @throws FeaturestoreException
 * @throws SQLException
 */
public FeaturegroupDTO enableFeaturegroupOnline(Featurestore featurestore, Featuregroup featuregroup, Project project, Users user) throws FeaturestoreException, SQLException, ServiceException, KafkaException, SchemaException, ProjectException, UserException, IOException, HopsSecurityException {
    if (!settings.isOnlineFeaturestore()) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_ONLINE_NOT_ENABLED, Level.FINE, "Online Featurestore is not enabled for this Hopsworks cluster.");
    }
    if (!onlineFeaturestoreController.checkIfDatabaseExists(onlineFeaturestoreController.getOnlineFeaturestoreDbName(featurestore.getProject()))) {
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_ONLINE_NOT_ENABLED, Level.FINE, "Online Featurestore is not enabled for this project. To enable online feature store, talk to an " + "administrator.");
    }
    CachedFeaturegroup cachedFeaturegroup = featuregroup.getCachedFeaturegroup();
    List<FeatureGroupFeatureDTO> features = getFeaturesDTO(featuregroup, project, user);
    if (cachedFeaturegroup.getTimeTravelFormat() == TimeTravelFormat.HUDI) {
        features = dropHudiSpecFeatureGroupFeature(features);
    }
    if (!cachedFeaturegroup.isOnlineEnabled()) {
        onlineFeaturegroupController.setupOnlineFeatureGroup(featurestore, featuregroup, features, project, user);
    }
    // Set foreign key of the cached feature group to the new online feature group
    cachedFeaturegroup.setOnlineEnabled(true);
    cachedFeatureGroupFacade.updateMetadata(cachedFeaturegroup);
    return convertCachedFeaturegroupToDTO(featuregroup, project, user);
}

Example 2 with CachedFeaturegroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.CachedFeaturegroup in project hopsworks by logicalclocks.

the class FeaturegroupController method createFeaturegroupNoValidation.

public FeaturegroupDTO createFeaturegroupNoValidation(Featurestore featurestore, FeaturegroupDTO featuregroupDTO, Project project, Users user) throws FeaturestoreException, SQLException, ProvenanceException, ServiceException, KafkaException, SchemaException, ProjectException, UserException, IOException, HopsSecurityException {
    // Persist specific feature group metadata (cached fg or on-demand fg)
    OnDemandFeaturegroup onDemandFeaturegroup = null;
    CachedFeaturegroup cachedFeaturegroup = null;
    List<FeatureGroupFeatureDTO> featuresNoHudi = null;
    if (featuregroupDTO instanceof CachedFeaturegroupDTO) {
        // make copy of schema without hudi columns
        featuresNoHudi = new ArrayList<>(featuregroupDTO.getFeatures());
        cachedFeaturegroup = cachedFeaturegroupController.createCachedFeaturegroup(featurestore, (CachedFeaturegroupDTO) featuregroupDTO, project, user);
    } else {
        onDemandFeaturegroup = onDemandFeaturegroupController.createOnDemandFeaturegroup(featurestore, (OnDemandFeaturegroupDTO) featuregroupDTO, project, user);
    }
    // Persist basic feature group metadata
    Featuregroup featuregroup = persistFeaturegroupMetadata(featurestore, user, featuregroupDTO, cachedFeaturegroup, onDemandFeaturegroup);
    // online feature group needs to be set up after persisting metadata in order to get feature group id
    if (featuregroupDTO instanceof CachedFeaturegroupDTO && settings.isOnlineFeaturestore() && featuregroup.getCachedFeaturegroup().isOnlineEnabled()) {
        onlineFeaturegroupController.setupOnlineFeatureGroup(featurestore, featuregroup, featuresNoHudi, project, user);
    }
    FeaturegroupDTO completeFeaturegroupDTO = convertFeaturegrouptoDTO(featuregroup, project, user);
    // Extract metadata
    String hdfsUsername = hdfsUsersController.getHdfsUserName(project, user);
    DistributedFileSystemOps udfso = dfs.getDfsOps(hdfsUsername);
    try {
        String fgPath = Utils.getFeaturestorePath(featurestore.getProject(), settings) + "/" + Utils.getFeaturegroupName(featuregroup);
        fsController.featuregroupAttachXAttrs(fgPath, completeFeaturegroupDTO, udfso);
    } finally {
        dfs.closeDfsClient(udfso);
    }
    // Log activity
    fsActivityFacade.logMetadataActivity(user, featuregroup, FeaturestoreActivityMeta.FG_CREATED, null);
    return completeFeaturegroupDTO;
}

Example 3 with CachedFeaturegroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.CachedFeaturegroup in project hopsworks by logicalclocks.

the class TestQueryController method setup.

@Before
public void setup() {
    fs = new Featurestore();
    fs.setHiveDbId(1l);
    fs.setProject(new Project("test_proj"));
    cachedFeaturegroup = new CachedFeaturegroup();
    cachedFeaturegroup.setTimeTravelFormat(TimeTravelFormat.NONE);
    fg1 = new Featuregroup(1);
    fg1.setName("fg1");
    fg1.setVersion(1);
    fg1.setCachedFeaturegroup(cachedFeaturegroup);
    fg1.setFeaturestore(fs);
    fg2 = new Featuregroup(2);
    fg2.setName("fg2");
    fg2.setVersion(1);
    fg2.setCachedFeaturegroup(cachedFeaturegroup);
    fg2.setFeaturestore(fs);
    fg3 = new Featuregroup(3);
    fg3.setName("fg3");
    fg3.setVersion(1);
    fg3.setCachedFeaturegroup(cachedFeaturegroup);
    fg3.setFeaturestore(fs);
    fg4 = new Featuregroup(4);
    fg4.setName("fg4");
    fg4.setVersion(1);
    fg4.setCachedFeaturegroup(cachedFeaturegroup);
    fg4.setFeaturestore(fs);
    fgHudi = new Featuregroup(5);
    fgHudi.setName("fgHudi");
    fgHudi.setVersion(1);
    hudiFeatureGroup = new CachedFeaturegroup();
    hudiFeatureGroup.setTimeTravelFormat(TimeTravelFormat.HUDI);
    fgHudi.setCachedFeaturegroup(hudiFeatureGroup);
    fgHudi.setFeaturestore(fs);
    fg1Features = new ArrayList<>();
    fg1Features.add(new Feature("pr", "", true));
    fg1Features.add(new Feature("fg1_ft2", "", false));
    fg1FeaturesDTO = new ArrayList<>();
    fg1FeaturesDTO.add(new FeatureGroupFeatureDTO("pr", "Integer", "", true, false, "", null));
    fg1FeaturesDTO.add(new FeatureGroupFeatureDTO("fg1_ft2", "String", "", false, false, "", null));
    fg2Features = new ArrayList<>();
    fg2Features.add(new Feature("pr", "", true));
    fg2Features.add(new Feature("fg2_ft2", "", false));
    fg2FeaturesDTO = new ArrayList<>();
    fg2FeaturesDTO.add(new FeatureGroupFeatureDTO("pr", "Integer", "", true, false, "", null));
    fg2FeaturesDTO.add(new FeatureGroupFeatureDTO("fg2_ft2", "String", "", false, false, "", null));
    fg3Features = new ArrayList<>();
    fg3Features.add(new Feature("fg3_ft1", "", true));
    fg3Features.add(new Feature("fg3_ft2", "", false));
    fg4Features = new ArrayList<>();
    fg4Features.add(new Feature("pr", "fg4", true));
    fg4Features.add(new Feature("fg4_ft4_1", "fg4", "Float", null, "prefix4_"));
    fg4Features.add(new Feature("fg4_ft4_2", "fg4", "Float", null, "prefix4_"));
    fg4Features.add(new Feature("_hoodie_record_key", "fg4", "String", null, null));
    fg4Features.add(new Feature("_hoodie_partition_path", "fg4", "String", null, null));
    fg4Features.add(new Feature("_hoodie_commit_time", "fg4", "String", null, null));
    fg4Features.add(new Feature("_hoodie_file_name", "fg4", "String", null, null));
    fg4Features.add(new Feature("_hoodie_commit_seqno", "fg4", "String", null, null));
    singleEqualsJoinOperator = Arrays.asList(SqlCondition.EQUALS);
    featuregroupController = Mockito.mock(FeaturegroupController.class);
    featuregroupFacade = Mockito.mock(FeaturegroupFacade.class);
    featurestoreFacade = Mockito.mock(FeaturestoreFacade.class);
    onlineFeaturestoreController = Mockito.mock(OnlineFeaturestoreController.class);
    cachedFeaturegroupController = Mockito.mock(CachedFeaturegroupController.class);
    project = Mockito.mock(Project.class);
    user = Mockito.mock(Users.class);
    filterController = new FilterController(new ConstructorController());
    target = new QueryController(featuregroupController, featuregroupFacade, filterController, featurestoreFacade, onlineFeaturestoreController);
    new JoinController(new ConstructorController());
}

Example 4 with CachedFeaturegroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.CachedFeaturegroup in project hopsworks by logicalclocks.

the class TestJoinController method setup.

@Before
public void setup() {
    fs = new Featurestore();
    fs.setHiveDbId(1l);
    fs.setProject(new Project("test_proj"));
    cachedFeaturegroup = new CachedFeaturegroup();
    cachedFeaturegroup.setTimeTravelFormat(TimeTravelFormat.NONE);
    fg1 = new Featuregroup(1);
    fg1.setName("fg1");
    fg1.setVersion(1);
    fg1.setCachedFeaturegroup(cachedFeaturegroup);
    fg1.setFeaturestore(fs);
    fg2 = new Featuregroup(2);
    fg2.setName("fg2");
    fg2.setVersion(1);
    fg2.setCachedFeaturegroup(cachedFeaturegroup);
    fg2.setFeaturestore(fs);
    joinController = new JoinController(new ConstructorController());
}

Example 5 with CachedFeaturegroup

use of io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.CachedFeaturegroup in project hopsworks by logicalclocks.

the class TestFeatureGroupCommitController method setup.

@Before
public void setup() {
    Inode inode = new Inode();
    HiveSds hiveSds = new HiveSds();
    hiveSds.setSdId(1l);
    hiveSds.setLocation("hopsfs://namenode.service.consul:8020/apps/hive/warehouse/test_proj_featurestore.db/fg1_1");
    hiveSds.setInode(inode);
    HiveTbls hiveTbls = new HiveTbls();
    hiveTbls.setSdId(hiveSds);
    hiveTbls.setTblName("fg1_1");
    CachedFeaturegroup cachedFeaturegroup = new CachedFeaturegroup();
    cachedFeaturegroup.setHiveTbls(hiveTbls);
    fs = new Featurestore();
    fs.setHiveDbId(1l);
    fs.setProject(new Project("test_proj"));
    fg1 = new Featuregroup(1);
    fg1.setName("fg1_1");
    fg1.setVersion(1);
    fg1.setFeaturestore(fs);
    fg1.setCachedFeaturegroup(cachedFeaturegroup);
}