Examples with Featurestore io.hops.hopsworks.persistence.entity.featurestore.Featurestore used on opensource projects

Example 11 with Featurestore

use of io.hops.hopsworks.persistence.entity.featurestore.Featurestore in project hopsworks by logicalclocks.

the class ProjectController method addServiceFeaturestore.

/**
 * Add the featurestore service to the project,
 * 1. create the hive database for the featurestore
 * 2. insert featurestore metadata in the hopsworks db
 * 3. create a hopsworks dataset for the featurestore
 * 4. create a directory in resources to store json configurations for feature import jobs.
 *
 * @param project the project to add the featurestore service for
 * @param user the user adding the service
 * @param dfso dfso
 */
private void addServiceFeaturestore(Project project, Users user, DistributedFileSystemOps dfso, ProvTypeDTO datasetProvCore) throws FeaturestoreException, ProjectException, UserException {
    String featurestoreName = featurestoreController.getOfflineFeaturestoreDbName(project);
    try {
        // Create HiveDB for the featurestore
        hiveController.createDatabase(featurestoreName, "Featurestore database for project: " + project.getName());
        // Store featurestore metadata in Hopsworks
        Dataset trainingDatasets = datasetController.getByProjectAndDsName(project, null, project.getName() + "_" + Settings.ServiceDataset.TRAININGDATASETS.getName());
        Featurestore featurestore = featurestoreController.createProjectFeatureStore(project, user, featurestoreName, trainingDatasets);
        // Create Hopsworks Dataset of the HiveDb
        hiveController.createDatasetDb(project, user, dfso, featurestoreName, DatasetType.FEATURESTORE, featurestore, datasetProvCore);
        // Register built-in transformation function.
        transformationFunctionController.registerBuiltInTransformationFunctions(user, project, featurestore);
    } catch (SQLException | IOException | ServiceDiscoveryException ex) {
        LOGGER.log(Level.SEVERE, RESTCodes.FeaturestoreErrorCode.COULD_NOT_CREATE_FEATURESTORE.getMessage(), ex);
        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.COULD_NOT_CREATE_FEATURESTORE, Level.SEVERE, "project: " + project.getName(), ex.getMessage(), ex);
    }
}

Example 12 with Featurestore

use of io.hops.hopsworks.persistence.entity.featurestore.Featurestore in project hopsworks by logicalclocks.

the class ProjectController method removeMemberFromTeam.

public void removeMemberFromTeam(Project project, Users userToBeRemoved) throws ProjectException, ServiceException, IOException, GenericException, JobException, HopsSecurityException, TensorBoardException, FeaturestoreException {
    ProjectTeam projectTeam = projectTeamFacade.findProjectTeam(project, userToBeRemoved);
    if (projectTeam == null) {
        throw new ProjectException(RESTCodes.ProjectErrorCode.TEAM_MEMBER_NOT_FOUND, Level.FINE, "project: " + project + ", user: " + userToBeRemoved.getEmail());
    }
    // Not able to remove project owner regardless of who is trying to remove the member
    if (project.getOwner().equals(userToBeRemoved)) {
        throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_OWNER_NOT_ALLOWED, Level.FINE);
    }
    projectTeamFacade.removeProjectTeam(project, userToBeRemoved);
    String hdfsUser = hdfsUsersController.getHdfsUserName(project, userToBeRemoved);
    YarnClientWrapper yarnClientWrapper = ycs.getYarnClientSuper(settings.getConfiguration());
    YarnClient client = yarnClientWrapper.getYarnClient();
    try {
        Set<String> hdfsUsers = new HashSet<>();
        hdfsUsers.add(hdfsUser);
        List<ApplicationReport> projectsApps = client.getApplications(null, hdfsUsers, null, EnumSet.of(YarnApplicationState.ACCEPTED, YarnApplicationState.NEW, YarnApplicationState.NEW_SAVING, YarnApplicationState.RUNNING, YarnApplicationState.SUBMITTED));
        // kill jupyter for this user
        JupyterProject jupyterProject = jupyterFacade.findByUser(hdfsUser);
        if (jupyterProject != null) {
            jupyterController.shutdown(project, hdfsUser, userToBeRemoved, jupyterProject.getSecret(), jupyterProject.getCid(), jupyterProject.getPort());
        }
        // kill running TB if any
        tensorBoardController.cleanup(project, userToBeRemoved);
        // kill all jobs run by this user.
        // kill jobs
        List<Jobs> running = jobFacade.getRunningJobs(project, hdfsUser);
        if (running != null && !running.isEmpty()) {
            for (Jobs job : running) {
                executionController.stop(job);
            }
        }
        // wait that log aggregation for the jobs finish
        for (ApplicationReport appReport : projectsApps) {
            FinalApplicationStatus finalState = appReport.getFinalApplicationStatus();
            while (finalState.equals(FinalApplicationStatus.UNDEFINED)) {
                client.killApplication(appReport.getApplicationId());
                appReport = client.getApplicationReport(appReport.getApplicationId());
                finalState = appReport.getFinalApplicationStatus();
            }
            YarnLogUtil.waitForLogAggregation(client, appReport.getApplicationId());
        }
    } catch (YarnException | IOException | InterruptedException e) {
        throw new ProjectException(RESTCodes.ProjectErrorCode.KILL_MEMBER_JOBS, Level.SEVERE, "project: " + project + ", user: " + userToBeRemoved, e.getMessage(), e);
    } finally {
        ycs.closeYarnClient(yarnClientWrapper);
    }
    // trigger project team role remove handlers
    ProjectTeamRoleHandler.runProjectTeamRoleRemoveMembersHandlers(projectTeamRoleHandlers, project, Collections.singletonList(userToBeRemoved));
    // Revoke privileges for online feature store
    if (projectServiceFacade.isServiceEnabledForProject(project, ProjectServiceEnum.FEATURESTORE)) {
        Featurestore featurestore = featurestoreController.getProjectFeaturestore(project);
        onlineFeaturestoreController.removeOnlineFeaturestoreUser(featurestore, userToBeRemoved);
    }
    kafkaController.removeProjectMemberFromTopics(project, userToBeRemoved);
    certificateMaterializer.forceRemoveLocalMaterial(userToBeRemoved.getUsername(), project.getName(), null, false);
    try {
        certificatesController.revokeUserSpecificCertificates(project, userToBeRemoved);
    } catch (HopsSecurityException ex) {
        if (ex.getErrorCode() != RESTCodes.SecurityErrorCode.CERTIFICATE_NOT_FOUND) {
            LOGGER.log(Level.SEVERE, "Could not delete certificates when removing member " + userToBeRemoved.getUsername() + " from project " + project.getName() + ". Manual cleanup is needed!!!", ex);
            throw ex;
        }
    } catch (IOException | GenericException ex) {
        LOGGER.log(Level.SEVERE, "Could not delete certificates when removing member " + userToBeRemoved.getUsername() + " from project " + project.getName() + ". Manual cleanup is needed!!!", ex);
        throw ex;
    }
    // TODO: projectTeam might be null?
    hdfsUsersController.removeMember(projectTeam);
}

Example 13 with Featurestore

use of io.hops.hopsworks.persistence.entity.featurestore.Featurestore in project hopsworks by logicalclocks.

the class TestAlertManagerConfigTimer method createAlertReceiver.

private AlertReceiver createAlertReceiver(Integer id, Receiver receiver, AlertType alertType) throws JsonProcessingException {
    AlertReceiver alertReceiver = new AlertReceiver();
    alertReceiver.setId(id);
    alertReceiver.setName(receiver.getName());
    alertReceiver.setConfig(toJson(receiver));
    Project project = new Project("project1");
    Jobs job = new Jobs();
    job.setProject(project);
    job.setName("job1");
    Featurestore featurestore = new Featurestore();
    featurestore.setProject(project);
    Featuregroup featuregroup = new Featuregroup();
    featuregroup.setFeaturestore(featurestore);
    featuregroup.setName("fg1");
    List<JobAlert> jobAlerts = new ArrayList<>();
    jobAlerts.add(new JobAlert(random.nextInt(1000), JobAlertStatus.FAILED, alertType, AlertSeverity.CRITICAL, new Date(), job, alertReceiver));
    jobAlerts.add(new JobAlert(random.nextInt(1000), JobAlertStatus.KILLED, alertType, AlertSeverity.WARNING, new Date(), job, alertReceiver));
    alertReceiver.setJobAlertCollection(jobAlerts);
    List<FeatureGroupAlert> featureGroupAlerts = new ArrayList<>();
    featureGroupAlerts.add(new FeatureGroupAlert(random.nextInt(1000), ValidationRuleAlertStatus.FAILURE, alertType, AlertSeverity.CRITICAL, new Date(), featuregroup, alertReceiver));
    featureGroupAlerts.add(new FeatureGroupAlert(random.nextInt(1000), ValidationRuleAlertStatus.WARNING, alertType, AlertSeverity.WARNING, new Date(), featuregroup, alertReceiver));
    alertReceiver.setFeatureGroupAlertCollection(featureGroupAlerts);
    List<ProjectServiceAlert> projectServiceAlerts = new ArrayList<>();
    projectServiceAlerts.add(new ProjectServiceAlert(random.nextInt(1000), ProjectServiceEnum.FEATURESTORE, ProjectServiceAlertStatus.VALIDATION_FAILURE, alertType, AlertSeverity.WARNING, new Date(), project, alertReceiver));
    projectServiceAlerts.add(new ProjectServiceAlert(random.nextInt(1000), ProjectServiceEnum.JOBS, ProjectServiceAlertStatus.JOB_FAILED, alertType, AlertSeverity.WARNING, new Date(), project, alertReceiver));
    alertReceiver.setProjectServiceAlertCollection(projectServiceAlerts);
    return alertReceiver;
}

Example 14 with Featurestore

use of io.hops.hopsworks.persistence.entity.featurestore.Featurestore in project hopsworks by logicalclocks.

the class FeatureStoreExpectationsResource method createExpectation.

@ApiOperation(value = "Create feature store expectation.", response = ExpectationDTO.class)
@PUT
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response createExpectation(Expectation expectation, @Context SecurityContext sc, @Context UriInfo uriInfo) throws FeaturestoreException {
    // Workaround to ignore the list initialized from a json like {legalValues: null}
    if (expectation.getRules() != null && !expectation.getRules().isEmpty()) {
        expectation.getRules().stream().filter(rule -> rule.getLegalValues() != null && rule.getLegalValues().size() == 1 && rule.getLegalValues().get(0) == null).forEach(rule -> rule.setLegalValues(null));
    }
    FeatureStoreExpectation featureStoreExpectation = featureGroupValidationsController.createOrUpdateExpectation(featurestore, new io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.Expectation(expectation.getName(), expectation.getDescription(), expectation.getFeatures(), expectation.getRules()));
    ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.EXPECTATIONS);
    ExpectationDTO dto = featureStoreExpectationsBuilder.build(uriInfo, resourceRequest, project, featurestore, featureStoreExpectation);
    return Response.ok().entity(dto).build();
}