Examples with ServiceException io.hops.hopsworks.exceptions.ServiceException used on opensource projects

Example 11 with ServiceException

use of io.hops.hopsworks.exceptions.ServiceException in project hopsworks by logicalclocks.

the class ElasticController method featurestoreSearch.

/**
 * @param searchTerm
 * @param docProjectIds - pe specific FEATUREGROUP, TRAININGDATASET, FEATURE. No ALL allowed
 * @param from
 * @param size
 * @return
 * @throws ElasticException
 * @throws ServiceException
 */
public Map<FeaturestoreDocType, SearchResponse> featurestoreSearch(String searchTerm, Map<FeaturestoreDocType, Set<Integer>> docProjectIds, int from, int size) throws ElasticException, ServiceException {
    // check if the indices are up and running
    if (!elasticClientCtrl.mngIndexExists(Settings.FEATURESTORE_INDEX)) {
        throw new ServiceException(RESTCodes.ServiceErrorCode.ELASTIC_INDEX_NOT_FOUND, Level.SEVERE, "index: " + Settings.FEATURESTORE_INDEX);
    }
    QueryBuilder fgQB = null;
    QueryBuilder tdQB = null;
    QueryBuilder fQB = null;
    List<Pair<QueryBuilder, HighlightBuilder>> qbs = new LinkedList<>();
    if (docProjectIds.containsKey(FeaturestoreDocType.FEATUREGROUP)) {
        fgQB = addProjectToQuery(featuregroupQueryB(searchTerm), docProjectIds.get(FeaturestoreDocType.FEATUREGROUP));
        qbs.add(Pair.with(fgQB, featuregroupHighlighter()));
    }
    if (docProjectIds.containsKey(FeaturestoreDocType.TRAININGDATASET)) {
        tdQB = addProjectToQuery(trainingdatasetQueryB(searchTerm), docProjectIds.get(FeaturestoreDocType.TRAININGDATASET));
        qbs.add(Pair.with(tdQB, trainingDatasetHighlighter()));
    }
    if (docProjectIds.containsKey(FeaturestoreDocType.FEATURE)) {
        fQB = addProjectToQuery(featureQueryB(searchTerm), docProjectIds.get(FeaturestoreDocType.FEATURE));
        qbs.add(Pair.with(fQB, featureHighlighter()));
    }
    MultiSearchResponse response = executeSearchQuery(qbs, from, size);
    Map<FeaturestoreDocType, SearchResponse> result = new HashMap<>();
    int idx = 0;
    if (docProjectIds.containsKey(FeaturestoreDocType.FEATUREGROUP)) {
        checkResponse(fgQB, response.getResponses()[idx].getResponse());
        result.put(FeaturestoreDocType.FEATUREGROUP, response.getResponses()[idx].getResponse());
        idx++;
    }
    if (docProjectIds.containsKey(FeaturestoreDocType.TRAININGDATASET)) {
        checkResponse(tdQB, response.getResponses()[idx].getResponse());
        result.put(FeaturestoreDocType.TRAININGDATASET, response.getResponses()[idx].getResponse());
        idx++;
    }
    if (docProjectIds.containsKey(FeaturestoreDocType.FEATURE)) {
        checkResponse(fQB, response.getResponses()[idx].getResponse());
        result.put(FeaturestoreDocType.FEATURE, response.getResponses()[idx].getResponse());
        idx++;
    }
    return result;
}

Example 12 with ServiceException

use of io.hops.hopsworks.exceptions.ServiceException in project hopsworks by logicalclocks.

the class LocalHostJupyterProcessMgr method stopJupyterServer.

@Override
@TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED)
public void stopJupyterServer(Project project, Users user, String hdfsUsername, String jupyterHomePath, String cid, Integer port) throws ServiceException {
    if (jupyterHomePath == null || cid == null || port == null) {
        throw new IllegalArgumentException("Invalid arguments when stopping the Jupyter Server.");
    }
    // 1. Remove jupyter settings from the DB for this notebook first. If this fails, keep going to kill the notebook
    try {
        jupyterFacade.remove(hdfsUsername, port);
    } catch (Exception e) {
        LOGGER.severe("Problem when removing jupyter notebook entry from jupyter_project table: " + jupyterHomePath);
    }
    // 2. Then kill the jupyter notebook server. If this step isn't
    String prog = settings.getSudoersDir() + "/jupyter.sh";
    if (jupyterHomePath.isEmpty()) {
        jupyterHomePath = "''";
    }
    int exitValue = 0;
    Integer id = 1;
    ProcessDescriptor.Builder pdBuilder = new ProcessDescriptor.Builder().addCommand("/usr/bin/sudo").addCommand(prog).addCommand("kill").addCommand(jupyterHomePath).addCommand(cid).addCommand(hdfsUsername).setWaitTimeout(10L, TimeUnit.SECONDS);
    if (!LOGGER.isLoggable(Level.FINE)) {
        pdBuilder.ignoreOutErrStreams(true);
    }
    try {
        ProcessResult processResult = osProcessExecutor.execute(pdBuilder.build());
        LOGGER.log(Level.FINE, processResult.getStdout());
        exitValue = processResult.getExitCode();
    } catch (IOException ex) {
        throw new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_STOP_ERROR, Level.SEVERE, "exitValue: " + exitValue, ex.getMessage(), ex);
    }
    if (exitValue != 0) {
        throw new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_STOP_ERROR, Level.SEVERE, "exitValue: " + exitValue);
    }
}

Example 13 with ServiceException

use of io.hops.hopsworks.exceptions.ServiceException in project hopsworks by logicalclocks.

the class LocalHostJupyterProcessMgr method startJupyterServer.

@Override
@TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED)
public JupyterDTO startJupyterServer(Project project, String secretConfig, String hdfsUser, Users user, JupyterSettings js, String allowOrigin) throws ServiceException, JobException {
    String prog = settings.getSudoersDir() + "/jupyter.sh";
    Integer port = ThreadLocalRandom.current().nextInt(40000, 59999);
    JupyterPaths jp = jupyterConfigFilesGenerator.generateConfiguration(project, secretConfig, hdfsUser, user, js, port, allowOrigin);
    String secretDir = settings.getStagingDir() + Settings.PRIVATE_DIRS + js.getSecret();
    String token = TokenGenerator.generateToken(TOKEN_LENGTH);
    String cid = "";
    // The Jupyter Notebook is running at: http://localhost:8888/?token=c8de56fa4deed24899803e93c227592aef6538f93025fe01
    int maxTries = 5;
    // kill any running servers for this user, clear cached entries
    while (maxTries > 0) {
        try {
            // use pidfile to kill any running servers
            ProcessDescriptor processDescriptor = new ProcessDescriptor.Builder().addCommand("/usr/bin/sudo").addCommand(prog).addCommand("start").addCommand(jp.getNotebookPath()).addCommand(settings.getHadoopSymbolicLinkDir() + "-" + settings.getHadoopVersion()).addCommand(hdfsUser).addCommand(settings.getAnacondaProjectDir()).addCommand(port.toString()).addCommand(HopsUtils.getJupyterLogName(hdfsUser, port)).addCommand(secretDir).addCommand(jp.getCertificatesDir()).addCommand(hdfsUser).addCommand(token).addCommand(js.getMode().getValue()).addCommand(projectUtils.getFullDockerImageName(project, false)).addCommand(Boolean.toString(js.isGitBackend())).redirectErrorStream(true).setCurrentWorkingDirectory(new File(jp.getNotebookPath())).setWaitTimeout(60L, TimeUnit.SECONDS).build();
            String pidfile = jp.getRunDirPath() + "/jupyter.pid";
            ProcessResult processResult = osProcessExecutor.execute(processDescriptor);
            if (processResult.getExitCode() != 0) {
                String errorMsg = "Could not start Jupyter server. Exit code: " + processResult.getExitCode() + " Error: stdout: " + processResult.getStdout() + " stderr: " + processResult.getStderr();
                LOGGER.log(Level.SEVERE, errorMsg);
                throw new IOException(errorMsg);
            }
            // Read the pid for Jupyter Notebook
            cid = com.google.common.io.Files.readFirstLine(new File(pidfile), Charset.defaultCharset());
            return new JupyterDTO(port, token, cid, secretConfig, jp.getCertificatesDir());
        } catch (Exception ex) {
            LOGGER.log(Level.SEVERE, "Problem executing shell script to start Jupyter server", ex);
            maxTries--;
        }
    }
    String errorMsg = "Failed to start Jupyter";
    throw new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_START_ERROR, Level.SEVERE, errorMsg, errorMsg + " for project " + project);
}

Example 14 with ServiceException

use of io.hops.hopsworks.exceptions.ServiceException in project hopsworks by logicalclocks.

the class ProjectController method createProject.

/**
 * Creates a new project(project), the related DIR, the different services in
 * the project, and the master of the
 * project.
 * <p>
 * This needs to be an atomic operation (all or nothing) REQUIRES_NEW will
 * make sure a new transaction is created even
 * if this method is called from within a transaction.
 *
 * @param projectDTO
 * @param owner
 * @param sessionId
 * @return
 */
public Project createProject(ProjectDTO projectDTO, Users owner, String sessionId) throws DatasetException, GenericException, KafkaException, ProjectException, UserException, HopsSecurityException, ServiceException, FeaturestoreException, ElasticException, SchemaException, IOException {
    Long startTime = System.currentTimeMillis();
    // check that the project name is ok
    String projectName = projectDTO.getProjectName();
    FolderNameValidator.isValidProjectName(projectUtils, projectName);
    List<ProjectServiceEnum> projectServices = new ArrayList<>();
    if (projectDTO.getServices() != null) {
        for (String s : projectDTO.getServices()) {
            ProjectServiceEnum se = ProjectServiceEnum.valueOf(s.toUpperCase());
            projectServices.add(se);
        }
    }
    LOGGER.log(Level.FINE, () -> "PROJECT CREATION TIME. Step 1: " + (System.currentTimeMillis() - startTime));
    DistributedFileSystemOps dfso = null;
    Project project = null;
    try {
        dfso = dfs.getDfsOps();
        /*
       * create a project in the database
       * if the creation go through it means that there is no other project with
       * the same name.
       * this project creation act like a lock, no other project can be created
       * with the same name
       * until this project is removed from the database
       */
        try {
            project = createProject(projectName, owner, projectDTO.getDescription(), dfso);
        } catch (EJBException ex) {
            LOGGER.log(Level.WARNING, null, ex);
            Path dummy = new Path("/tmp/" + projectName);
            try {
                dfso.rm(dummy, true);
            } catch (IOException e) {
                LOGGER.log(Level.SEVERE, null, e);
            }
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_EXISTS, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 2 (hdfs): {0}", System.currentTimeMillis() - startTime);
        verifyProject(project, dfso, sessionId);
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 3 (verify): {0}", System.currentTimeMillis() - startTime);
        // Run the handlers.
        try {
            ProjectHandler.runProjectPreCreateHandlers(projectHandlers, project);
        } catch (ProjectException ex) {
            cleanup(project, sessionId, null, true, owner);
            throw ex;
        }
        List<Future<?>> projectCreationFutures = new ArrayList<>();
        // This is an async call
        try {
            projectCreationFutures.add(certificatesController.generateCertificates(project, owner));
        } catch (Exception ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new HopsSecurityException(RESTCodes.SecurityErrorCode.CERT_CREATION_ERROR, Level.SEVERE, "project: " + project.getName() + "owner: " + owner.getUsername(), ex.getMessage(), ex);
        }
        String username = hdfsUsersController.getHdfsUserName(project, owner);
        if (username == null || username.isEmpty()) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new UserException(RESTCodes.UserErrorCode.USER_WAS_NOT_FOUND, Level.SEVERE, "project: " + project.getName() + "owner: " + owner.getUsername());
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 4 (certs): {0}", System.currentTimeMillis() - startTime);
        // all the verifications have passed, we can now create the project
        // create the project folder
        ProvTypeDTO provType = settings.getProvType().dto;
        try {
            mkProjectDIR(projectName, dfso);
            fsProvController.updateProjectProvType(project, provType, dfso);
        } catch (IOException | EJBException | ProvenanceException ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_FOLDER_NOT_CREATED, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 5 (folders): {0}", System.currentTimeMillis() - startTime);
        // update the project with the project folder inode
        try {
            setProjectInode(project, dfso);
        } catch (IOException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_INODE_CREATION_ERROR, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 6 (inodes): {0}", System.currentTimeMillis() - startTime);
        // set payment and quotas
        try {
            setProjectOwnerAndQuotas(project, dfso, owner);
        } catch (IOException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new ProjectException(RESTCodes.ProjectErrorCode.QUOTA_ERROR, Level.SEVERE, "project: " + project.getName(), ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 7 (quotas): {0}", System.currentTimeMillis() - startTime);
        try {
            hdfsUsersController.addProjectFolderOwner(project, dfso);
            createProjectLogResources(owner, project, dfso);
        } catch (IOException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_SET_PERMISSIONS_ERROR, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 8 (logs): {0}", System.currentTimeMillis() - startTime);
        // inconsistencies
        try {
            elasticController.deleteProjectIndices(project);
            elasticController.deleteProjectSavedObjects(projectName);
            LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 9 (elastic cleanup): {0}", System.currentTimeMillis() - startTime);
        } catch (ElasticException ex) {
            LOGGER.log(Level.FINE, "Error while cleaning old project indices", ex);
        }
        logProject(project, OperationType.Add);
        // enable services
        for (ProjectServiceEnum service : projectServices) {
            try {
                projectCreationFutures.addAll(addService(project, service, owner, dfso, provType));
            } catch (RESTException | IOException ex) {
                cleanup(project, sessionId, projectCreationFutures);
                throw ex;
            }
        }
        try {
            for (Future f : projectCreationFutures) {
                if (f != null) {
                    f.get();
                }
            }
        } catch (InterruptedException | ExecutionException ex) {
            LOGGER.log(Level.SEVERE, "Error while waiting for the certificate generation thread to finish. Will try to " + "cleanup...", ex);
            cleanup(project, sessionId, projectCreationFutures);
            throw new HopsSecurityException(RESTCodes.SecurityErrorCode.CERT_CREATION_ERROR, Level.SEVERE);
        }
        // Run the handlers.
        try {
            ProjectHandler.runProjectPostCreateHandlers(projectHandlers, project);
        } catch (ProjectException ex) {
            cleanup(project, sessionId, projectCreationFutures);
            throw ex;
        }
        try {
            project = environmentController.createEnv(project, owner);
        } catch (PythonException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_ANACONDA_ENABLE_ERROR, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 10 (env): {0}", System.currentTimeMillis() - startTime);
        return project;
    } finally {
        if (dfso != null) {
            dfso.close();
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 11 (close): {0}", System.currentTimeMillis() - startTime);
    }
}

Example 15 with ServiceException

use of io.hops.hopsworks.exceptions.ServiceException in project hopsworks by logicalclocks.

the class JupyterController method getNotebookRelativeFilePath.

public String getNotebookRelativeFilePath(String hdfsUser, String sessionKernelId, DistributedFileSystemOps udfso) throws ServiceException {
    String relativeNotebookPath = null;
    JupyterProject jp = jupyterFacade.findByUser(hdfsUser);
    JSONArray sessionsArray = new JSONArray(ClientBuilder.newClient().target("http://" + jupyterManager.getJupyterHost() + ":" + jp.getPort() + "/hopsworks-api/jupyter/" + jp.getPort() + "/api/sessions?token=" + jp.getToken()).request().method("GET").readEntity(String.class));
    boolean foundKernel = false;
    for (int i = 0; i < sessionsArray.length(); i++) {
        JSONObject session = (JSONObject) sessionsArray.get(i);
        JSONObject kernel = (JSONObject) session.get("kernel");
        String kernelId = kernel.getString("id");
        if (kernelId.equals(sessionKernelId)) {
            relativeNotebookPath = session.getString("path");
            foundKernel = true;
        }
    }
    if (!foundKernel) {
        throw new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_NOTEBOOK_VERSIONING_FAILED, Level.FINE, "failed to find kernel " + sessionKernelId);
    }
    return relativeNotebookPath;
}