Examples with JupyterSettings io.hops.hopsworks.persistence.entity.jupyter.JupyterSettings used on opensource projects

Example 1 with JupyterSettings

use of io.hops.hopsworks.persistence.entity.jupyter.JupyterSettings in project hopsworks by logicalclocks.

the class JupyterService method getGitStatusOfJupyterRepo.

@GET
@Path("/git/status")
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response getGitStatusOfJupyterRepo(@Context SecurityContext sc) throws ProjectException, ServiceException {
    Users user = jWTHelper.getUserPrincipal(sc);
    String projectUser = hdfsUsersController.getHdfsUserName(project, user);
    JupyterProject jupyterProject = jupyterFacade.findByUser(projectUser);
    if (jupyterProject == null) {
        throw new ProjectException(RESTCodes.ProjectErrorCode.JUPYTER_SERVER_NOT_FOUND, Level.FINE, "Could not found Jupyter server", "Could not found Jupyter server for Hopsworks user: " + projectUser);
    }
    if (!jupyterManager.ping(jupyterProject)) {
        throw new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_SERVERS_NOT_RUNNING, Level.FINE, "Jupyter server is not running", "Jupyter server for Hopsworks user: " + projectUser + " is not running");
    }
    JupyterSettings jupyterSettings = jupyterSettingsFacade.findByProjectUser(project, user.getEmail());
    RepositoryStatus status = NullJupyterNbVCSController.EMPTY_REPOSITORY_STATUS;
    if (jupyterSettings.isGitBackend()) {
        status = jupyterNbVCSController.status(jupyterProject, jupyterSettings);
    }
    return Response.ok(status).build();
}

Example 2 with JupyterSettings

use of io.hops.hopsworks.persistence.entity.jupyter.JupyterSettings in project hopsworks by logicalclocks.

the class JupyterService method settings.

/**
 * Get livy session Yarn AppId
 *
 * @param sc
 * @return
 */
@GET
@Path("/settings")
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response settings(@Context SecurityContext sc) {
    Users user = jWTHelper.getUserPrincipal(sc);
    JupyterSettings js = jupyterSettingsFacade.findByProjectUser(project, user.getEmail());
    if (settings.isPythonKernelEnabled()) {
        js.setPrivateDir(settings.getStagingDir() + Settings.PRIVATE_DIRS + js.getSecret());
    }
    js.setGitAvailable(jupyterNbVCSController.isGitAvailable());
    js.setMode(JupyterMode.JUPYTER_LAB);
    return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(js).build();
}

Example 3 with JupyterSettings

use of io.hops.hopsworks.persistence.entity.jupyter.JupyterSettings in project hopsworks by logicalclocks.

the class JupyterJWTManager method recover.

protected void recover() {
    LOG.log(INFO, "Starting Jupyter JWT manager recovery");
    List<MaterializedJWT> failed2recover = new ArrayList<>();
    // Get state from the database
    for (MaterializedJWT materializedJWT : materializedJWTFacade.findAll4Jupyter()) {
        LOG.log(Level.FINEST, "Recovering Jupyter JWT " + materializedJWT.getIdentifier());
        // First lookup project and user in db
        Project project = projectFacade.find(materializedJWT.getIdentifier().getProjectId());
        Users user = userFacade.find(materializedJWT.getIdentifier().getUserId());
        if (project == null || user == null) {
            LOG.log(Level.WARNING, "Tried to recover " + materializedJWT.getIdentifier() + " but could not find " + "either Project or User");
            failed2recover.add(materializedJWT);
            continue;
        }
        // Get Jupyter configuration from db
        String hdfsUsername = hdfsUsersController.getHdfsUserName(project, user);
        JupyterProject jupyterProject = jupyterFacade.findByUser(hdfsUsername);
        if (jupyterProject == null) {
            LOG.log(Level.FINEST, "There is no Jupyter configuration persisted for " + materializedJWT.getIdentifier());
            failed2recover.add(materializedJWT);
            continue;
        }
        // Check if Jupyter is still running
        if (!jupyterManager.ping(jupyterProject)) {
            LOG.log(Level.FINEST, "Jupyter server is not running for " + materializedJWT.getIdentifier() + " Skip recovering...");
            failed2recover.add(materializedJWT);
            continue;
        }
        JupyterSettings jupyterSettings = jupyterSettingsFacade.findByProjectUser(project, user.getEmail());
        Path tokenFile = constructTokenFilePath(jupyterSettings);
        String token = null;
        JupyterJWT jupyterJWT = null;
        CidAndPort pidAndPort = new CidAndPort(jupyterProject.getCid(), jupyterProject.getPort());
        try {
            token = FileUtils.readFileToString(tokenFile.toFile());
            DecodedJWT decodedJWT = jwtController.verifyToken(token, settings.getJWTIssuer());
            jupyterJWT = new JupyterJWT(project, user, DateUtils.date2LocalDateTime(decodedJWT.getExpiresAt()), pidAndPort);
            jupyterJWT.token = token;
            jupyterJWT.tokenFile = tokenFile;
            LOG.log(Level.FINE, "Successfully read existing JWT from local filesystem");
        } catch (IOException | JWTException | JWTDecodeException ex) {
            LOG.log(Level.FINE, "Could not recover Jupyter JWT from local filesystem, generating new!", ex);
            // JWT does not exist or it is not valid any longer
            // We should create a new one
            String[] audience = new String[] { "api" };
            LocalDateTime expirationDate = LocalDateTime.now().plus(settings.getJWTLifetimeMs(), ChronoUnit.MILLIS);
            String[] userRoles = usersController.getUserRoles(user).toArray(new String[1]);
            try {
                Map<String, Object> claims = new HashMap<>(3);
                claims.put(Constants.RENEWABLE, false);
                claims.put(Constants.EXPIRY_LEEWAY, settings.getJWTExpLeewaySec());
                claims.put(Constants.ROLES, userRoles);
                token = jwtController.createToken(settings.getJWTSigningKeyName(), false, settings.getJWTIssuer(), audience, DateUtils.localDateTime2Date(expirationDate), DateUtils.localDateTime2Date(DateUtils.getNow()), user.getUsername(), claims, SignatureAlgorithm.valueOf(settings.getJWTSignatureAlg()));
                jupyterJWT = new JupyterJWT(project, user, expirationDate, pidAndPort);
                jupyterJWT.token = token;
                jupyterJWT.tokenFile = tokenFile;
                jwtTokenWriter.writeToken(settings, jupyterJWT);
                LOG.log(Level.FINE, "Generated new Jupyter JWT cause could not recover existing");
            } catch (IOException recIOEx) {
                LOG.log(Level.WARNING, "Failed to recover Jupyter JWT for " + materializedJWT.getIdentifier() + ", generated new valid JWT but failed to write to local filesystem. Invalidating new token!" + " Continue recovering...");
                if (token != null) {
                    try {
                        jwtController.invalidate(token);
                    } catch (InvalidationException jwtInvEx) {
                    // NO-OP
                    }
                }
                failed2recover.add(materializedJWT);
                continue;
            } catch (GeneralSecurityException | JWTException jwtEx) {
                LOG.log(Level.WARNING, "Failed to recover Jupyter JWT for " + materializedJWT.getIdentifier() + ", tried to generate new token and it failed as well. Could not recover! Continue recovering...");
                // Did our best, it's good to know when you should give up
                failed2recover.add(materializedJWT);
                continue;
            }
        }
        addToken(jupyterJWT);
    }
    // Remove from the database entries that we failed to recover
    for (MaterializedJWT failedRecovery : failed2recover) {
        materializedJWTFacade.delete(failedRecovery.getIdentifier());
    }
    LOG.log(INFO, "Finished Jupyter JWT recovery");
}

Example 4 with JupyterSettings

use of io.hops.hopsworks.persistence.entity.jupyter.JupyterSettings in project hopsworks by logicalclocks.

the class JupyterSettingsFacade method findByProjectUser.

public JupyterSettings findByProjectUser(Project project, String email) {
    JupyterSettingsPK pk = new JupyterSettingsPK(project.getId(), email);
    JupyterSettings js;
    js = em.find(JupyterSettings.class, pk);
    if (js == null) {
        String secret = DigestUtils.sha256Hex(Integer.toString(ThreadLocalRandom.current().nextInt()));
        js = new JupyterSettings(pk);
        js.setSecret(secret);
        js.setJobConfig(new SparkJobConfiguration(ExperimentType.EXPERIMENT));
        js.setBaseDir(Utils.getProjectPath(project.getName()) + Settings.ServiceDataset.JUPYTER.getName());
        persist(js);
    }
    if (js.getJobConfig() == null) {
        js.setJobConfig(new SparkJobConfiguration(ExperimentType.EXPERIMENT));
    }
    return js;
}

Example 5 with JupyterSettings

use of io.hops.hopsworks.persistence.entity.jupyter.JupyterSettings in project hopsworks by logicalclocks.

the class NotebookBuilder method buildNotebook.

private void buildNotebook(SearchHit hit, NotebookDTO notebooks) {
    NotebookDTO item = new NotebookDTO();
    Map xattrMap = (Map) hit.getSourceAsMap().get("xattr");
    Long inodeId = Long.parseLong(hit.getId());
    Inode inode = inodeFacade.findById(inodeId);
    if (inode != null) {
        item.setPath(inodeController.getPath(inode));
    }
    Map jupyterConfigMap = (Map) xattrMap.get(Settings.META_NOTEBOOK_JUPYTER_CONFIG_XATTR_NAME);
    if (jupyterConfigMap != null && !jupyterConfigMap.isEmpty()) {
        try {
            String jupyterSettingsStr = jupyterConfigMap.get(Settings.META_NOTEBOOK_JUPYTER_CONFIG_XATTR_NAME).toString();
            item.setJupyterSettings(objectMapper.readValue(jupyterSettingsStr, JupyterSettings.class));
        } catch (IOException e) {
            LOGGER.log(Level.SEVERE, "Could not parse JupyterSettings for notebook at path: " + item.getPath(), e);
        }
        item.setDate(new Date(Long.parseLong(jupyterConfigMap.get(Settings.META_USAGE_TIME).toString())));
    }
    notebooks.addItem(item);
}