use of io.hops.hopsworks.persistence.entity.jupyter.JupyterProject in project hopsworks by logicalclocks.
the class JupyterService method getGitStatusOfJupyterRepo.
@GET
@Path("/git/status")
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response getGitStatusOfJupyterRepo(@Context SecurityContext sc) throws ProjectException, ServiceException {
Users user = jWTHelper.getUserPrincipal(sc);
String projectUser = hdfsUsersController.getHdfsUserName(project, user);
JupyterProject jupyterProject = jupyterFacade.findByUser(projectUser);
if (jupyterProject == null) {
throw new ProjectException(RESTCodes.ProjectErrorCode.JUPYTER_SERVER_NOT_FOUND, Level.FINE, "Could not found Jupyter server", "Could not found Jupyter server for Hopsworks user: " + projectUser);
}
if (!jupyterManager.ping(jupyterProject)) {
throw new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_SERVERS_NOT_RUNNING, Level.FINE, "Jupyter server is not running", "Jupyter server for Hopsworks user: " + projectUser + " is not running");
}
JupyterSettings jupyterSettings = jupyterSettingsFacade.findByProjectUser(project, user.getEmail());
RepositoryStatus status = NullJupyterNbVCSController.EMPTY_REPOSITORY_STATUS;
if (jupyterSettings.isGitBackend()) {
status = jupyterNbVCSController.status(jupyterProject, jupyterSettings);
}
return Response.ok(status).build();
}
use of io.hops.hopsworks.persistence.entity.jupyter.JupyterProject in project hopsworks by logicalclocks.
the class JupyterService method startNotebookServer.
@POST
@Path("/start")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response startNotebookServer(JupyterSettings jupyterSettings, @Context HttpServletRequest req, @Context SecurityContext sc, @Context UriInfo uriInfo) throws ProjectException, HopsSecurityException, ServiceException, GenericException, JobException {
Users hopsworksUser = jWTHelper.getUserPrincipal(sc);
String hdfsUser = hdfsUsersController.getHdfsUserName(project, hopsworksUser);
// from in the front-end
if (jupyterSettings.getUsers() == null) {
jupyterSettings.setUsers(hopsworksUser);
}
if (project.getPaymentType().equals(PaymentType.PREPAID)) {
YarnProjectsQuota projectQuota = yarnProjectsQuotaFacade.findByProjectName(project.getName());
if (projectQuota == null || projectQuota.getQuotaRemaining() <= 0) {
throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_QUOTA_ERROR, Level.FINE);
}
}
if (project.getPythonEnvironment() == null) {
throw new ProjectException(RESTCodes.ProjectErrorCode.ANACONDA_NOT_ENABLED, Level.FINE);
}
if (jupyterSettings.getMode() == null) {
// set default mode for jupyter if mode is null
jupyterSettings.setMode(JupyterMode.JUPYTER_LAB);
}
// Jupyter Git works only for JupyterLab
if (jupyterSettings.isGitBackend() && jupyterSettings.getMode().equals(JupyterMode.JUPYTER_CLASSIC)) {
throw new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_START_ERROR, Level.FINE, "Git support available only in JupyterLab");
}
// Do not allow auto push on shutdown if api key is missing
GitConfig gitConfig = jupyterSettings.getGitConfig();
if (jupyterSettings.isGitBackend() && gitConfig.getShutdownAutoPush() && Strings.isNullOrEmpty(gitConfig.getApiKeyName())) {
throw new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_START_ERROR, Level.FINE, "Auto push not supported if api key is not configured.");
}
// Verify that API token has got write access on the repo if ShutdownAutoPush is enabled
if (jupyterSettings.isGitBackend() && gitConfig.getShutdownAutoPush() && !jupyterNbVCSController.hasWriteAccess(hopsworksUser, gitConfig.getApiKeyName(), gitConfig.getRemoteGitURL(), gitConfig.getGitBackend())) {
throw new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_START_ERROR, Level.FINE, "API token " + gitConfig.getApiKeyName() + " does not have write access on " + gitConfig.getRemoteGitURL());
}
JupyterProject jp = jupyterFacade.findByUser(hdfsUser);
if (jp == null) {
HdfsUsers user = hdfsUsersFacade.findByName(hdfsUser);
String configSecret = DigestUtils.sha256Hex(Integer.toString(ThreadLocalRandom.current().nextInt()));
JupyterDTO dto = null;
DistributedFileSystemOps dfso = dfsService.getDfsOps();
String allowOriginHost = uriInfo.getBaseUri().getHost();
int allowOriginPort = uriInfo.getBaseUri().getPort();
String allowOriginPortStr = allowOriginPort != -1 ? ":" + allowOriginPort : "";
String allowOrigin = settings.getJupyterOriginScheme() + "://" + allowOriginHost + allowOriginPortStr;
try {
jupyterSettingsFacade.update(jupyterSettings);
// Inspect dependencies
sparkController.inspectDependencies(project, hopsworksUser, (SparkJobConfiguration) jupyterSettings.getJobConfig());
dto = jupyterManager.startJupyterServer(project, configSecret, hdfsUser, hopsworksUser, jupyterSettings, allowOrigin);
jupyterJWTManager.materializeJWT(hopsworksUser, project, jupyterSettings, dto.getCid(), dto.getPort(), JUPYTER_JWT_AUD);
HopsUtils.materializeCertificatesForUserCustomDir(project.getName(), user.getUsername(), settings.getHdfsTmpCertDir(), dfso, certificateMaterializer, settings, dto.getCertificatesDir());
jupyterManager.waitForStartup(project, hopsworksUser);
} catch (ServiceException | TimeoutException ex) {
if (dto != null) {
jupyterController.shutdownQuietly(project, hdfsUser, hopsworksUser, configSecret, dto.getCid(), dto.getPort());
}
throw new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_START_ERROR, Level.SEVERE, ex.getMessage(), null, ex);
} catch (IOException ex) {
if (dto != null) {
jupyterController.shutdownQuietly(project, hdfsUser, hopsworksUser, configSecret, dto.getCid(), dto.getPort());
}
throw new HopsSecurityException(RESTCodes.SecurityErrorCode.CERT_MATERIALIZATION_ERROR, Level.SEVERE, ex.getMessage(), null, ex);
} finally {
if (dfso != null) {
dfsService.closeDfsClient(dfso);
}
}
String externalIp = Ip.getHost(req.getRequestURL().toString());
try {
Date expirationDate = new Date();
Calendar cal = Calendar.getInstance();
cal.setTime(expirationDate);
cal.add(Calendar.HOUR_OF_DAY, jupyterSettings.getShutdownLevel());
expirationDate = cal.getTime();
jp = jupyterFacade.saveServer(externalIp, project, configSecret, dto.getPort(), user.getId(), dto.getToken(), dto.getCid(), expirationDate, jupyterSettings.isNoLimit());
// set minutes left until notebook server is killed
Duration durationLeft = Duration.between(new Date().toInstant(), jp.getExpires().toInstant());
jp.setMinutesUntilExpiration(durationLeft.toMinutes());
} catch (Exception e) {
LOGGER.log(Level.SEVERE, "Failed to save Jupyter notebook settings", e);
jupyterController.shutdownQuietly(project, hdfsUser, hopsworksUser, configSecret, dto.getCid(), dto.getPort());
}
if (jp == null) {
throw new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_SAVE_SETTINGS_ERROR, Level.SEVERE);
}
if (jupyterSettings.isGitBackend()) {
try {
// Init is idempotent, calling it on an already initialized repo won't affect it
jupyterNbVCSController.init(jp, jupyterSettings);
if (jupyterSettings.getGitConfig().getStartupAutoPull()) {
jupyterNbVCSController.pull(jp, jupyterSettings);
}
} catch (ServiceException ex) {
jupyterController.shutdownQuietly(project, hdfsUser, hopsworksUser, configSecret, dto.getCid(), dto.getPort());
throw ex;
}
}
} else {
throw new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_SERVER_ALREADY_RUNNING, Level.FINE);
}
return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(jp).build();
}
use of io.hops.hopsworks.persistence.entity.jupyter.JupyterProject in project hopsworks by logicalclocks.
the class JupyterService method isRunning.
@GET
@Path("/running")
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response isRunning(@Context HttpServletRequest req, @Context SecurityContext sc) throws ServiceException {
String hdfsUser = getHdfsUser(sc);
JupyterProject jp = jupyterFacade.findByUser(hdfsUser);
if (jp == null) {
throw new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_SERVERS_NOT_FOUND, Level.FINE);
}
// Check to make sure the jupyter notebook server is running
boolean running = jupyterManager.ping(jp);
// we should remove the DB entry (and restart the notebook server).
if (!running) {
jupyterFacade.remove(hdfsUser, jp.getPort());
throw new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_SERVERS_NOT_RUNNING, Level.FINE);
}
// set minutes left until notebook server is killed
Duration durationLeft = Duration.between(new Date().toInstant(), jp.getExpires().toInstant());
jp.setMinutesUntilExpiration(durationLeft.toMinutes());
return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(jp).build();
}
use of io.hops.hopsworks.persistence.entity.jupyter.JupyterProject in project hopsworks by logicalclocks.
the class JupyterNotebookCleaner method execute.
@Timeout
@TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED)
public void execute(Timer timer) {
try {
LOGGER.log(Level.INFO, "Running JupyterNotebookCleaner.");
// 1. Get all Running Jupyter Notebook Servers
List<JupyterProject> servers = jupyterFacade.getAllNotebookServers();
if (servers != null && !servers.isEmpty()) {
Date currentDate = Calendar.getInstance().getTime();
for (JupyterProject jp : servers) {
// If the notebook is expired
if (!jp.isNoLimit() && jp.getExpires().before(currentDate)) {
try {
HdfsUsers hdfsUser = hdfsUsersFacade.find(jp.getHdfsUserId());
Users user = usersFacade.findByUsername(hdfsUser.getUsername());
LOGGER.log(Level.FINE, "Shutting down expired notebook for hdfs user " + hdfsUser.getName());
jupyterController.shutdown(jp.getProjectId(), hdfsUser.getName(), user, jp.getSecret(), jp.getCid(), jp.getPort());
} catch (Exception e) {
LOGGER.log(Level.SEVERE, "Failed to cleanup notebook with port " + jp.getPort(), e);
}
}
}
}
} catch (Exception e) {
LOGGER.log(Level.SEVERE, "Got an exception while cleaning up jupyter notebooks");
}
}
use of io.hops.hopsworks.persistence.entity.jupyter.JupyterProject in project hopsworks by logicalclocks.
the class JupyterController method getNotebookRelativeFilePath.
public String getNotebookRelativeFilePath(String hdfsUser, String sessionKernelId, DistributedFileSystemOps udfso) throws ServiceException {
String relativeNotebookPath = null;
JupyterProject jp = jupyterFacade.findByUser(hdfsUser);
JSONArray sessionsArray = new JSONArray(ClientBuilder.newClient().target("http://" + jupyterManager.getJupyterHost() + ":" + jp.getPort() + "/hopsworks-api/jupyter/" + jp.getPort() + "/api/sessions?token=" + jp.getToken()).request().method("GET").readEntity(String.class));
boolean foundKernel = false;
for (int i = 0; i < sessionsArray.length(); i++) {
JSONObject session = (JSONObject) sessionsArray.get(i);
JSONObject kernel = (JSONObject) session.get("kernel");
String kernelId = kernel.getString("id");
if (kernelId.equals(sessionKernelId)) {
relativeNotebookPath = session.getString("path");
foundKernel = true;
}
}
if (!foundKernel) {
throw new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_NOTEBOOK_VERSIONING_FAILED, Level.FINE, "failed to find kernel " + sessionKernelId);
}
return relativeNotebookPath;
}
Aggregations