Examples with Featurestore io.hops.hopsworks.persistence.entity.featurestore.Featurestore used on opensource projects

Example 1 with Featurestore

use of io.hops.hopsworks.persistence.entity.featurestore.Featurestore in project hopsworks by logicalclocks.

the class TestCommitBuilder method setup.

@Before
public void setup() {
    featureGroupCommitController = Mockito.mock(FeatureGroupCommitController.class);
    Mockito.when(featureGroupCommitController.getCommitDetails(Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anySet(), Mockito.anySet())).thenReturn(new AbstractFacade.CollectionInfo(0L, new ArrayList()));
    // throws exception if limit or offset is null
    Mockito.when(featureGroupCommitController.getCommitDetails(Mockito.anyInt(), Mockito.anyInt(), Mockito.isNull(), Mockito.anySet(), Mockito.anySet())).thenThrow(new NullPointerException());
    Mockito.when(featureGroupCommitController.getCommitDetails(Mockito.anyInt(), Mockito.isNull(), Mockito.anyInt(), Mockito.anySet(), Mockito.anySet())).thenThrow(new NullPointerException());
    commitBuilder = new CommitBuilder(featureGroupCommitController);
    mockUriInfo = Mockito.mock(UriInfo.class);
    UriBuilder mockUriBuilder = Mockito.mock(UriBuilder.class);
    Mockito.when(mockUriBuilder.path(Mockito.anyString())).thenReturn(mockUriBuilder);
    Mockito.when(mockUriInfo.getBaseUriBuilder()).thenReturn(mockUriBuilder);
    featuregroup = new Featuregroup();
    featuregroup.setId(0);
    Featurestore featurestore = new Featurestore();
    featurestore.setId(0);
    featuregroup.setFeaturestore(featurestore);
    project = new Project();
    project.setId(0);
}

Example 2 with Featurestore

use of io.hops.hopsworks.persistence.entity.featurestore.Featurestore in project hopsworks by logicalclocks.

the class FeaturestoreController method createProjectFeatureStore.

/**
 * Creates a new featurestore in the database
 *
 * @param project                 project of the new featurestore
 * @param featurestoreName        the name of the new featurestore
 * @param trainingDatasetsFolder  the Hopsworks dataset where training datasets are stored by default
 * @return the created featurestore
 * @throws FeaturestoreException
 */
public Featurestore createProjectFeatureStore(Project project, Users user, String featurestoreName, Dataset trainingDatasetsFolder) throws FeaturestoreException, ProjectException, UserException {
    // Get HiveDbId for the newly created Hive featurestore DB
    Long hiveDbId = featurestoreFacade.getHiveDatabaseId(featurestoreName);
    // Store featurestore metadata in Hopsworks
    Featurestore featurestore = new Featurestore();
    featurestore.setProject(project);
    featurestore.setHiveDbId(hiveDbId);
    featurestore.setCreated(new Date());
    featurestoreFacade.persist(featurestore);
    activityFacade.persistActivity(ActivityFacade.CREATED_FEATURESTORE + featurestoreName, project, project.getOwner(), ActivityFlag.SERVICE);
    activityFacade.persistActivity(ActivityFacade.ADDED_FEATURESTORE_STORAGE_CONNECTOR + getOfflineFeaturestoreDbName(project), project, project.getOwner(), ActivityFlag.SERVICE);
    featurestoreStorageConnectorController.createStorageConnector(user, project, featurestore, hopsfsTrainingDatasetConnector(trainingDatasetsFolder));
    activityFacade.persistActivity(ActivityFacade.ADDED_FEATURESTORE_STORAGE_CONNECTOR + trainingDatasetsFolder.getName(), project, project.getOwner(), ActivityFlag.SERVICE);
    featurestoreStorageConnectorController.createStorageConnector(user, project, featurestore, createOfflineJdbcConnector(featurestoreName));
    activityFacade.persistActivity(ActivityFacade.ADDED_FEATURESTORE_STORAGE_CONNECTOR + project.getName(), project, project.getOwner(), ActivityFlag.SERVICE);
    createOnlineFeatureStore(project, user, featurestore);
    return featurestore;
}

Example 3 with Featurestore

use of io.hops.hopsworks.persistence.entity.featurestore.Featurestore in project hopsworks by logicalclocks.

the class ProjectController method addMember.

public boolean addMember(ProjectTeam projectTeam, Project project, Users newMember, Users owner, DistributedFileSystemOps dfso) throws UserException, KafkaException, ProjectException, FeaturestoreException, IOException {
    if (projectTeam.getTeamRole() == null || (!projectTeam.getTeamRole().equals(ProjectRoleTypes.DATA_SCIENTIST.getRole()) && !projectTeam.getTeamRole().equals(ProjectRoleTypes.DATA_OWNER.getRole()))) {
        projectTeam.setTeamRole(ProjectRoleTypes.DATA_SCIENTIST.getRole());
    }
    projectTeam.setTimestamp(new Date());
    if (newMember != null && !projectTeamFacade.isUserMemberOfProject(project, newMember)) {
        // this makes sure that the member is added to the project sent as the
        // first param b/c the security check was made on the parameter sent as path.
        projectTeam.getProjectTeamPK().setProjectId(project.getId());
        projectTeam.setProject(project);
        projectTeam.setUser(newMember);
        project.getProjectTeamCollection().add(projectTeam);
        projectFacade.update(project);
        hdfsUsersController.addNewProjectMember(projectTeam, dfso);
        // Add user to kafka topics ACLs by default
        if (projectServicesFacade.isServiceEnabledForProject(project, ProjectServiceEnum.KAFKA)) {
            kafkaController.addProjectMemberToTopics(project, newMember.getEmail());
        }
        // if online-featurestore service is enabled in the project, give new member access to it
        if (projectServiceFacade.isServiceEnabledForProject(project, ProjectServiceEnum.FEATURESTORE) && settings.isOnlineFeaturestore()) {
            Featurestore featurestore = featurestoreController.getProjectFeaturestore(project);
            onlineFeaturestoreController.createDatabaseUser(projectTeam.getUser(), featurestore, projectTeam.getTeamRole());
        }
        // TODO: This should now be a REST call
        Future<CertificatesController.CertsResult> certsResultFuture = null;
        try {
            certsResultFuture = certificatesController.generateCertificates(project, newMember);
            certsResultFuture.get();
        } catch (Exception ex) {
            try {
                if (certsResultFuture != null) {
                    certsResultFuture.get();
                }
                certificatesController.revokeUserSpecificCertificates(project, newMember);
            } catch (IOException | InterruptedException | ExecutionException | HopsSecurityException | GenericException e) {
                String failedUser = project.getName() + HdfsUsersController.USER_NAME_DELIMITER + newMember.getUsername();
                LOGGER.log(Level.SEVERE, "Could not delete user certificates for user " + failedUser + ". Manual cleanup is needed!!! ", e);
            }
            LOGGER.log(Level.SEVERE, "error while creating certificates, jupyter kernel: " + ex.getMessage(), ex);
            hdfsUsersController.removeMember(projectTeam);
            projectTeamFacade.removeProjectTeam(project, newMember);
            throw new EJBException("Could not create certificates for user");
        }
        // trigger project team role update handlers
        ProjectTeamRoleHandler.runProjectTeamRoleAddMembersHandlers(projectTeamRoleHandlers, project, Collections.singletonList(newMember), ProjectRoleTypes.fromString(projectTeam.getTeamRole()), false);
        String message = "You have been added to project " + project.getName() + " with a role " + projectTeam.getTeamRole() + ".";
        messageController.send(newMember, owner, "You have been added to a project.", message, message, "");
        LOGGER.log(Level.FINE, "{0} - member added to project : {1}.", new Object[] { newMember.getEmail(), project.getName() });
        logActivity(ActivityFacade.NEW_MEMBER + projectTeam.getProjectTeamPK().getTeamMember(), owner, project, ActivityFlag.MEMBER);
        return true;
    } else {
        return false;
    }
}

Example 4 with Featurestore

use of io.hops.hopsworks.persistence.entity.featurestore.Featurestore in project hopsworks by logicalclocks.

the class TestQueryController method setup.

@Before
public void setup() {
    fs = new Featurestore();
    fs.setHiveDbId(1l);
    fs.setProject(new Project("test_proj"));
    cachedFeaturegroup = new CachedFeaturegroup();
    cachedFeaturegroup.setTimeTravelFormat(TimeTravelFormat.NONE);
    fg1 = new Featuregroup(1);
    fg1.setName("fg1");
    fg1.setVersion(1);
    fg1.setCachedFeaturegroup(cachedFeaturegroup);
    fg1.setFeaturestore(fs);
    fg2 = new Featuregroup(2);
    fg2.setName("fg2");
    fg2.setVersion(1);
    fg2.setCachedFeaturegroup(cachedFeaturegroup);
    fg2.setFeaturestore(fs);
    fg3 = new Featuregroup(3);
    fg3.setName("fg3");
    fg3.setVersion(1);
    fg3.setCachedFeaturegroup(cachedFeaturegroup);
    fg3.setFeaturestore(fs);
    fg4 = new Featuregroup(4);
    fg4.setName("fg4");
    fg4.setVersion(1);
    fg4.setCachedFeaturegroup(cachedFeaturegroup);
    fg4.setFeaturestore(fs);
    fgHudi = new Featuregroup(5);
    fgHudi.setName("fgHudi");
    fgHudi.setVersion(1);
    hudiFeatureGroup = new CachedFeaturegroup();
    hudiFeatureGroup.setTimeTravelFormat(TimeTravelFormat.HUDI);
    fgHudi.setCachedFeaturegroup(hudiFeatureGroup);
    fgHudi.setFeaturestore(fs);
    fg1Features = new ArrayList<>();
    fg1Features.add(new Feature("pr", "", true));
    fg1Features.add(new Feature("fg1_ft2", "", false));
    fg1FeaturesDTO = new ArrayList<>();
    fg1FeaturesDTO.add(new FeatureGroupFeatureDTO("pr", "Integer", "", true, false, "", null));
    fg1FeaturesDTO.add(new FeatureGroupFeatureDTO("fg1_ft2", "String", "", false, false, "", null));
    fg2Features = new ArrayList<>();
    fg2Features.add(new Feature("pr", "", true));
    fg2Features.add(new Feature("fg2_ft2", "", false));
    fg2FeaturesDTO = new ArrayList<>();
    fg2FeaturesDTO.add(new FeatureGroupFeatureDTO("pr", "Integer", "", true, false, "", null));
    fg2FeaturesDTO.add(new FeatureGroupFeatureDTO("fg2_ft2", "String", "", false, false, "", null));
    fg3Features = new ArrayList<>();
    fg3Features.add(new Feature("fg3_ft1", "", true));
    fg3Features.add(new Feature("fg3_ft2", "", false));
    fg4Features = new ArrayList<>();
    fg4Features.add(new Feature("pr", "fg4", true));
    fg4Features.add(new Feature("fg4_ft4_1", "fg4", "Float", null, "prefix4_"));
    fg4Features.add(new Feature("fg4_ft4_2", "fg4", "Float", null, "prefix4_"));
    fg4Features.add(new Feature("_hoodie_record_key", "fg4", "String", null, null));
    fg4Features.add(new Feature("_hoodie_partition_path", "fg4", "String", null, null));
    fg4Features.add(new Feature("_hoodie_commit_time", "fg4", "String", null, null));
    fg4Features.add(new Feature("_hoodie_file_name", "fg4", "String", null, null));
    fg4Features.add(new Feature("_hoodie_commit_seqno", "fg4", "String", null, null));
    singleEqualsJoinOperator = Arrays.asList(SqlCondition.EQUALS);
    featuregroupController = Mockito.mock(FeaturegroupController.class);
    featuregroupFacade = Mockito.mock(FeaturegroupFacade.class);
    featurestoreFacade = Mockito.mock(FeaturestoreFacade.class);
    onlineFeaturestoreController = Mockito.mock(OnlineFeaturestoreController.class);
    cachedFeaturegroupController = Mockito.mock(CachedFeaturegroupController.class);
    project = Mockito.mock(Project.class);
    user = Mockito.mock(Users.class);
    filterController = new FilterController(new ConstructorController());
    target = new QueryController(featuregroupController, featuregroupFacade, filterController, featurestoreFacade, onlineFeaturestoreController);
    new JoinController(new ConstructorController());
}

Example 5 with Featurestore

use of io.hops.hopsworks.persistence.entity.featurestore.Featurestore in project hopsworks by logicalclocks.

the class TestJoinController method setup.

@Before
public void setup() {
    fs = new Featurestore();
    fs.setHiveDbId(1l);
    fs.setProject(new Project("test_proj"));
    cachedFeaturegroup = new CachedFeaturegroup();
    cachedFeaturegroup.setTimeTravelFormat(TimeTravelFormat.NONE);
    fg1 = new Featuregroup(1);
    fg1.setName("fg1");
    fg1.setVersion(1);
    fg1.setCachedFeaturegroup(cachedFeaturegroup);
    fg1.setFeaturestore(fs);
    fg2 = new Featuregroup(2);
    fg2.setName("fg2");
    fg2.setVersion(1);
    fg2.setCachedFeaturegroup(cachedFeaturegroup);
    fg2.setFeaturestore(fs);
    joinController = new JoinController(new ConstructorController());
}