Search in sources :

Example 16 with JupyterProject

use of io.hops.hopsworks.persistence.entity.jupyter.JupyterProject in project hopsworks by logicalclocks.

the class ProjectController method removeMemberFromTeam.

public void removeMemberFromTeam(Project project, Users userToBeRemoved) throws ProjectException, ServiceException, IOException, GenericException, JobException, HopsSecurityException, TensorBoardException, FeaturestoreException {
    ProjectTeam projectTeam = projectTeamFacade.findProjectTeam(project, userToBeRemoved);
    if (projectTeam == null) {
        throw new ProjectException(RESTCodes.ProjectErrorCode.TEAM_MEMBER_NOT_FOUND, Level.FINE, "project: " + project + ", user: " + userToBeRemoved.getEmail());
    }
    // Not able to remove project owner regardless of who is trying to remove the member
    if (project.getOwner().equals(userToBeRemoved)) {
        throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_OWNER_NOT_ALLOWED, Level.FINE);
    }
    projectTeamFacade.removeProjectTeam(project, userToBeRemoved);
    String hdfsUser = hdfsUsersController.getHdfsUserName(project, userToBeRemoved);
    YarnClientWrapper yarnClientWrapper = ycs.getYarnClientSuper(settings.getConfiguration());
    YarnClient client = yarnClientWrapper.getYarnClient();
    try {
        Set<String> hdfsUsers = new HashSet<>();
        hdfsUsers.add(hdfsUser);
        List<ApplicationReport> projectsApps = client.getApplications(null, hdfsUsers, null, EnumSet.of(YarnApplicationState.ACCEPTED, YarnApplicationState.NEW, YarnApplicationState.NEW_SAVING, YarnApplicationState.RUNNING, YarnApplicationState.SUBMITTED));
        // kill jupyter for this user
        JupyterProject jupyterProject = jupyterFacade.findByUser(hdfsUser);
        if (jupyterProject != null) {
            jupyterController.shutdown(project, hdfsUser, userToBeRemoved, jupyterProject.getSecret(), jupyterProject.getCid(), jupyterProject.getPort());
        }
        // kill running TB if any
        tensorBoardController.cleanup(project, userToBeRemoved);
        // kill all jobs run by this user.
        // kill jobs
        List<Jobs> running = jobFacade.getRunningJobs(project, hdfsUser);
        if (running != null && !running.isEmpty()) {
            for (Jobs job : running) {
                executionController.stop(job);
            }
        }
        // wait that log aggregation for the jobs finish
        for (ApplicationReport appReport : projectsApps) {
            FinalApplicationStatus finalState = appReport.getFinalApplicationStatus();
            while (finalState.equals(FinalApplicationStatus.UNDEFINED)) {
                client.killApplication(appReport.getApplicationId());
                appReport = client.getApplicationReport(appReport.getApplicationId());
                finalState = appReport.getFinalApplicationStatus();
            }
            YarnLogUtil.waitForLogAggregation(client, appReport.getApplicationId());
        }
    } catch (YarnException | IOException | InterruptedException e) {
        throw new ProjectException(RESTCodes.ProjectErrorCode.KILL_MEMBER_JOBS, Level.SEVERE, "project: " + project + ", user: " + userToBeRemoved, e.getMessage(), e);
    } finally {
        ycs.closeYarnClient(yarnClientWrapper);
    }
    // trigger project team role remove handlers
    ProjectTeamRoleHandler.runProjectTeamRoleRemoveMembersHandlers(projectTeamRoleHandlers, project, Collections.singletonList(userToBeRemoved));
    // Revoke privileges for online feature store
    if (projectServiceFacade.isServiceEnabledForProject(project, ProjectServiceEnum.FEATURESTORE)) {
        Featurestore featurestore = featurestoreController.getProjectFeaturestore(project);
        onlineFeaturestoreController.removeOnlineFeaturestoreUser(featurestore, userToBeRemoved);
    }
    kafkaController.removeProjectMemberFromTopics(project, userToBeRemoved);
    certificateMaterializer.forceRemoveLocalMaterial(userToBeRemoved.getUsername(), project.getName(), null, false);
    try {
        certificatesController.revokeUserSpecificCertificates(project, userToBeRemoved);
    } catch (HopsSecurityException ex) {
        if (ex.getErrorCode() != RESTCodes.SecurityErrorCode.CERTIFICATE_NOT_FOUND) {
            LOGGER.log(Level.SEVERE, "Could not delete certificates when removing member " + userToBeRemoved.getUsername() + " from project " + project.getName() + ". Manual cleanup is needed!!!", ex);
            throw ex;
        }
    } catch (IOException | GenericException ex) {
        LOGGER.log(Level.SEVERE, "Could not delete certificates when removing member " + userToBeRemoved.getUsername() + " from project " + project.getName() + ". Manual cleanup is needed!!!", ex);
        throw ex;
    }
    // TODO: projectTeam might be null?
    hdfsUsersController.removeMember(projectTeam);
}
Also used : FinalApplicationStatus(org.apache.hadoop.yarn.api.records.FinalApplicationStatus) JupyterProject(io.hops.hopsworks.persistence.entity.jupyter.JupyterProject) IOException(java.io.IOException) GenericException(io.hops.hopsworks.exceptions.GenericException) YarnClient(org.apache.hadoop.yarn.client.api.YarnClient) YarnException(org.apache.hadoop.yarn.exceptions.YarnException) HopsSecurityException(io.hops.hopsworks.exceptions.HopsSecurityException) ProjectException(io.hops.hopsworks.exceptions.ProjectException) ApplicationReport(org.apache.hadoop.yarn.api.records.ApplicationReport) ProjectTeam(io.hops.hopsworks.persistence.entity.project.team.ProjectTeam) Featurestore(io.hops.hopsworks.persistence.entity.featurestore.Featurestore) Jobs(io.hops.hopsworks.persistence.entity.jobs.description.Jobs) YarnClientWrapper(io.hops.hopsworks.common.yarn.YarnClientWrapper) HashSet(java.util.HashSet)

Example 17 with JupyterProject

use of io.hops.hopsworks.persistence.entity.jupyter.JupyterProject in project hopsworks by logicalclocks.

the class JupyterController method versionProgram.

public void versionProgram(String hdfsUser, String sessionKernelId, Path outputPath, DistributedFileSystemOps udfso) throws ServiceException {
    JupyterProject jp = jupyterFacade.findByUser(hdfsUser);
    String relativeNotebookPath = null;
    try {
        relativeNotebookPath = getNotebookRelativeFilePath(hdfsUser, sessionKernelId, udfso);
        // Get the content of the notebook should work in both spark and python kernels irregardless of contents manager
        if (!Strings.isNullOrEmpty(relativeNotebookPath)) {
            JSONObject notebookContents = new JSONObject(ClientBuilder.newClient().target("http://" + jupyterManager.getJupyterHost() + ":" + jp.getPort() + "/hopsworks-api/jupyter/" + jp.getPort() + "/api/contents/" + relativeNotebookPath + "?content=1&token=" + jp.getToken()).request().method("GET").readEntity(String.class));
            JSONObject notebookJSON = (JSONObject) notebookContents.get("content");
            try {
                udfso.create(outputPath, notebookJSON.toString());
            } catch (IOException e) {
                throw new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_NOTEBOOK_VERSIONING_FAILED, Level.FINE, "failed to save notebook content", e.getMessage(), e);
            }
        }
    } catch (Exception e) {
        throw new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_NOTEBOOK_VERSIONING_FAILED, Level.FINE, "failed to version notebook", e.getMessage(), e);
    }
}
Also used : JSONObject(org.json.JSONObject) ServiceException(io.hops.hopsworks.exceptions.ServiceException) JupyterProject(io.hops.hopsworks.persistence.entity.jupyter.JupyterProject) IOException(java.io.IOException) ServiceDiscoveryException(com.logicalclocks.servicediscoverclient.exceptions.ServiceDiscoveryException) IOException(java.io.IOException) ServiceException(io.hops.hopsworks.exceptions.ServiceException)

Aggregations

JupyterProject (io.hops.hopsworks.persistence.entity.jupyter.JupyterProject)17 HdfsUsers (io.hops.hopsworks.persistence.entity.hdfs.user.HdfsUsers)7 ServiceException (io.hops.hopsworks.exceptions.ServiceException)6 Users (io.hops.hopsworks.persistence.entity.user.Users)6 AllowedProjectRoles (io.hops.hopsworks.api.filter.AllowedProjectRoles)4 ProjectException (io.hops.hopsworks.exceptions.ProjectException)4 JWTRequired (io.hops.hopsworks.jwt.annotation.JWTRequired)4 IOException (java.io.IOException)4 Date (java.util.Date)4 Path (javax.ws.rs.Path)4 Produces (javax.ws.rs.Produces)4 JupyterSettings (io.hops.hopsworks.persistence.entity.jupyter.JupyterSettings)3 NoResultException (javax.persistence.NoResultException)3 GET (javax.ws.rs.GET)3 DistributedFileSystemOps (io.hops.hopsworks.common.hdfs.DistributedFileSystemOps)2 GenericException (io.hops.hopsworks.exceptions.GenericException)2 HopsSecurityException (io.hops.hopsworks.exceptions.HopsSecurityException)2 File (java.io.File)2 Duration (java.time.Duration)2 ArrayList (java.util.ArrayList)2