Examples with PythonException io.hops.hopsworks.exceptions.PythonException used on opensource projects

Example 1 with PythonException

use of io.hops.hopsworks.exceptions.PythonException in project hopsworks by logicalclocks.

the class ExperimentsResource method post.

@ApiOperation(value = "Create or update an experiment", response = ExperimentDTO.class)
@PUT
@Path("{id}")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response post(@PathParam("id") String id, ExperimentDTO experimentDTO, @QueryParam("type") ExperimentOperationType type, @Context HttpServletRequest req, @Context UriInfo uriInfo, @Context SecurityContext sc) throws DatasetException, ProvenanceException, PythonException, MetadataException, ProjectException, GenericException, ExperimentsException {
    if (experimentDTO == null) {
        throw new IllegalArgumentException("No Experiment configuration was provided");
    }
    Users user = jwtHelper.getUserPrincipal(sc);
    Project experimentProject = project;
    switch(type) {
        case INIT:
            {
                String experimentPath = Utils.getProjectPath(project.getName()) + Settings.HOPS_EXPERIMENTS_DATASET + "/" + id + "/" + Settings.ENVIRONMENT_FILE;
                experimentDTO.setEnvironment(environmentController.exportEnv(experimentProject, user, experimentPath));
                try {
                    String program = experimentsController.versionProgram(experimentProject, user, experimentDTO.getJobName(), experimentDTO.getKernelId(), id);
                    experimentDTO.setProgram(program);
                } catch (Exception e) {
                    LOGGER.log(Level.SEVERE, "Could not version notebook " + e.getMessage());
                }
            }
            break;
        case MODEL_UPDATE:
            {
                Project modelProject = getModelsProjectAndCheckAccess(experimentDTO);
                experimentsController.attachModel(user, experimentProject, id, modelProject, experimentDTO.getModel());
            }
            break;
        case FULL_UPDATE:
            {
            // no need to update the summary in any way
            }
            break;
        default:
            {
                throw new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.INFO, "unhandled experiment summary operation type:" + type);
            }
    }
    experimentsController.attachExperiment(user, experimentProject, id, experimentDTO);
    UriBuilder builder = uriInfo.getAbsolutePathBuilder().path(id);
    switch(type) {
        case INIT:
            return Response.created(builder.build()).entity(experimentDTO).build();
        case MODEL_UPDATE:
        case FULL_UPDATE:
            return Response.ok(builder.build()).entity(experimentDTO).build();
        default:
            {
                throw new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.INFO, "unhandled experiment summary operation type:" + type);
            }
    }
}

Example 2 with PythonException

use of io.hops.hopsworks.exceptions.PythonException in project hopsworks by logicalclocks.

the class LibraryResource method install.

@ApiOperation(value = "Install a python library in the environment")
@POST
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@Path("{library}")
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.PYTHON }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response install(LibrarySpecification librarySpecification, @PathParam("library") String library, @Context UriInfo uriInfo, @Context HttpServletRequest req, @Context SecurityContext sc) throws ServiceException, GenericException, PythonException, DatasetException {
    Users user = jwtHelper.getUserPrincipal(req);
    environmentController.checkCondaEnabled(project, pythonVersion, true);
    PackageSource packageSource = librarySpecification.getPackageSource();
    if (packageSource == null) {
        throw new PythonException(RESTCodes.PythonErrorCode.INSTALL_TYPE_NOT_SUPPORTED, Level.FINE);
    }
    switch(packageSource) {
        case PIP:
            validateLibrary(librarySpecification, library);
            librarySpecification.setChannelUrl("pypi");
            break;
        case CONDA:
            validateLibrary(librarySpecification, library);
            break;
        case EGG:
        case WHEEL:
        case REQUIREMENTS_TXT:
        case ENVIRONMENT_YAML:
            validateBundledDependency(user, librarySpecification);
            break;
        case GIT:
            validateGitURL(librarySpecification.getDependencyUrl());
            break;
        default:
            throw new PythonException(RESTCodes.PythonErrorCode.INSTALL_TYPE_NOT_SUPPORTED, Level.FINE);
    }
    environmentController.checkCondaEnvExists(project, user);
    PythonDep dep = libraryController.installLibrary(project, user, CondaInstallType.valueOf(packageSource.name().toUpperCase()), librarySpecification.getChannelUrl(), library, librarySpecification.getVersion(), librarySpecification.getDependencyUrl(), librarySpecification.getGitBackend(), librarySpecification.getGitApiKey());
    ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.LIBRARIES);
    LibraryDTO libraryDTO = librariesBuilder.build(uriInfo, resourceRequest, dep, project);
    return Response.created(libraryDTO.getHref()).entity(libraryDTO).build();
}

Example 3 with PythonException

use of io.hops.hopsworks.exceptions.PythonException in project hopsworks by logicalclocks.

the class LibraryResource method validateLibrary.

private void validateLibrary(LibrarySpecification librarySpecification, String library) throws PythonException {
    String version = librarySpecification.getVersion();
    PackageSource packageSource = librarySpecification.getPackageSource();
    String channel = librarySpecification.getChannelUrl();
    if (packageSource.equals(PackageSource.CONDA)) {
        if (channel == null) {
            throw new PythonException(RESTCodes.PythonErrorCode.CONDA_INSTALL_REQUIRES_CHANNEL, Level.FINE);
        } else {
            validateChannel(channel);
        }
    }
    validatePattern(library);
    if (!Strings.isNullOrEmpty(version)) {
        validatePattern(version);
    }
}

Example 4 with PythonException

use of io.hops.hopsworks.exceptions.PythonException in project hopsworks by logicalclocks.

the class ProjectController method createProject.

/**
 * Creates a new project(project), the related DIR, the different services in
 * the project, and the master of the
 * project.
 * <p>
 * This needs to be an atomic operation (all or nothing) REQUIRES_NEW will
 * make sure a new transaction is created even
 * if this method is called from within a transaction.
 *
 * @param projectDTO
 * @param owner
 * @param sessionId
 * @return
 */
public Project createProject(ProjectDTO projectDTO, Users owner, String sessionId) throws DatasetException, GenericException, KafkaException, ProjectException, UserException, HopsSecurityException, ServiceException, FeaturestoreException, ElasticException, SchemaException, IOException {
    Long startTime = System.currentTimeMillis();
    // check that the project name is ok
    String projectName = projectDTO.getProjectName();
    FolderNameValidator.isValidProjectName(projectUtils, projectName);
    List<ProjectServiceEnum> projectServices = new ArrayList<>();
    if (projectDTO.getServices() != null) {
        for (String s : projectDTO.getServices()) {
            ProjectServiceEnum se = ProjectServiceEnum.valueOf(s.toUpperCase());
            projectServices.add(se);
        }
    }
    LOGGER.log(Level.FINE, () -> "PROJECT CREATION TIME. Step 1: " + (System.currentTimeMillis() - startTime));
    DistributedFileSystemOps dfso = null;
    Project project = null;
    try {
        dfso = dfs.getDfsOps();
        /*
       * create a project in the database
       * if the creation go through it means that there is no other project with
       * the same name.
       * this project creation act like a lock, no other project can be created
       * with the same name
       * until this project is removed from the database
       */
        try {
            project = createProject(projectName, owner, projectDTO.getDescription(), dfso);
        } catch (EJBException ex) {
            LOGGER.log(Level.WARNING, null, ex);
            Path dummy = new Path("/tmp/" + projectName);
            try {
                dfso.rm(dummy, true);
            } catch (IOException e) {
                LOGGER.log(Level.SEVERE, null, e);
            }
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_EXISTS, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 2 (hdfs): {0}", System.currentTimeMillis() - startTime);
        verifyProject(project, dfso, sessionId);
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 3 (verify): {0}", System.currentTimeMillis() - startTime);
        // Run the handlers.
        try {
            ProjectHandler.runProjectPreCreateHandlers(projectHandlers, project);
        } catch (ProjectException ex) {
            cleanup(project, sessionId, null, true, owner);
            throw ex;
        }
        List<Future<?>> projectCreationFutures = new ArrayList<>();
        // This is an async call
        try {
            projectCreationFutures.add(certificatesController.generateCertificates(project, owner));
        } catch (Exception ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new HopsSecurityException(RESTCodes.SecurityErrorCode.CERT_CREATION_ERROR, Level.SEVERE, "project: " + project.getName() + "owner: " + owner.getUsername(), ex.getMessage(), ex);
        }
        String username = hdfsUsersController.getHdfsUserName(project, owner);
        if (username == null || username.isEmpty()) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new UserException(RESTCodes.UserErrorCode.USER_WAS_NOT_FOUND, Level.SEVERE, "project: " + project.getName() + "owner: " + owner.getUsername());
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 4 (certs): {0}", System.currentTimeMillis() - startTime);
        // all the verifications have passed, we can now create the project
        // create the project folder
        ProvTypeDTO provType = settings.getProvType().dto;
        try {
            mkProjectDIR(projectName, dfso);
            fsProvController.updateProjectProvType(project, provType, dfso);
        } catch (IOException | EJBException | ProvenanceException ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_FOLDER_NOT_CREATED, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 5 (folders): {0}", System.currentTimeMillis() - startTime);
        // update the project with the project folder inode
        try {
            setProjectInode(project, dfso);
        } catch (IOException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_INODE_CREATION_ERROR, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 6 (inodes): {0}", System.currentTimeMillis() - startTime);
        // set payment and quotas
        try {
            setProjectOwnerAndQuotas(project, dfso, owner);
        } catch (IOException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new ProjectException(RESTCodes.ProjectErrorCode.QUOTA_ERROR, Level.SEVERE, "project: " + project.getName(), ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 7 (quotas): {0}", System.currentTimeMillis() - startTime);
        try {
            hdfsUsersController.addProjectFolderOwner(project, dfso);
            createProjectLogResources(owner, project, dfso);
        } catch (IOException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_SET_PERMISSIONS_ERROR, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 8 (logs): {0}", System.currentTimeMillis() - startTime);
        // inconsistencies
        try {
            elasticController.deleteProjectIndices(project);
            elasticController.deleteProjectSavedObjects(projectName);
            LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 9 (elastic cleanup): {0}", System.currentTimeMillis() - startTime);
        } catch (ElasticException ex) {
            LOGGER.log(Level.FINE, "Error while cleaning old project indices", ex);
        }
        logProject(project, OperationType.Add);
        // enable services
        for (ProjectServiceEnum service : projectServices) {
            try {
                projectCreationFutures.addAll(addService(project, service, owner, dfso, provType));
            } catch (RESTException | IOException ex) {
                cleanup(project, sessionId, projectCreationFutures);
                throw ex;
            }
        }
        try {
            for (Future f : projectCreationFutures) {
                if (f != null) {
                    f.get();
                }
            }
        } catch (InterruptedException | ExecutionException ex) {
            LOGGER.log(Level.SEVERE, "Error while waiting for the certificate generation thread to finish. Will try to " + "cleanup...", ex);
            cleanup(project, sessionId, projectCreationFutures);
            throw new HopsSecurityException(RESTCodes.SecurityErrorCode.CERT_CREATION_ERROR, Level.SEVERE);
        }
        // Run the handlers.
        try {
            ProjectHandler.runProjectPostCreateHandlers(projectHandlers, project);
        } catch (ProjectException ex) {
            cleanup(project, sessionId, projectCreationFutures);
            throw ex;
        }
        try {
            project = environmentController.createEnv(project, owner);
        } catch (PythonException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_ANACONDA_ENABLE_ERROR, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 10 (env): {0}", System.currentTimeMillis() - startTime);
        return project;
    } finally {
        if (dfso != null) {
            dfso.close();
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 11 (close): {0}", System.currentTimeMillis() - startTime);
    }
}

Example 5 with PythonException

use of io.hops.hopsworks.exceptions.PythonException in project hopsworks by logicalclocks.

the class LibraryInstaller method createNewImage.

private void createNewImage(CondaCommands cc) throws IOException, ServiceDiscoveryException {
    File baseDir = new File("/tmp/docker/" + cc.getProjectId().getName());
    baseDir.mkdirs();
    try {
        File home = new File(System.getProperty("user.home"));
        File condarc = new File(home, ".condarc");
        File pip = new File(home, ".pip");
        FileUtils.copyFileToDirectory(condarc, baseDir);
        FileUtils.copyDirectoryToDirectory(pip, baseDir);
        File dockerFile = new File(baseDir, "dockerFile_" + cc.getProjectId().getName());
        try (BufferedWriter writer = new BufferedWriter(new FileWriter(dockerFile))) {
            String baseImage = projectUtils.getFullBaseImageName();
            // image that does not contain a python environment
            if (!Strings.isNullOrEmpty(cc.getEnvironmentFile()) && !cc.getInstallJupyter()) {
                baseImage = baseImage.replace(settings.getBaseDockerImagePythonName(), settings.getBaseNonPythonDockerImage());
            }
            writer.write("# syntax=docker/dockerfile:experimental");
            writer.newLine();
            writer.write("FROM " + baseImage);
            writer.newLine();
            // copy it in the image, create the env and delete it
            if (!Strings.isNullOrEmpty(cc.getEnvironmentFile())) {
                writer.write("RUN rm -f /root/.condarc");
                writer.newLine();
                // Materialize IMPORT FILE
                String environmentFilePath = cc.getEnvironmentFile();
                String environmentFile = FilenameUtils.getName(environmentFilePath);
                writer.write("COPY .condarc .pip " + environmentFile + " /root/");
                writer.newLine();
                copyCondaArtifactToLocal(environmentFilePath, baseDir + File.separator + environmentFile);
                if (environmentFilePath.endsWith(".yml")) {
                    if (cc.getInstallJupyter()) {
                        writer.write("RUN conda env update -f /root/" + environmentFile + " -n " + settings.getCurrentCondaEnvironment());
                    } else {
                        writer.write("RUN conda env create -f /root/" + environmentFile + " -p " + anaconda_project_dir);
                    }
                } else if (environmentFilePath.endsWith("/requirements.txt")) {
                    if (cc.getInstallJupyter()) {
                        writer.write("RUN pip install -r /root/" + environmentFile);
                    } else {
                        writer.write("RUN conda create -y -p " + anaconda_project_dir + " python=" + settings.getDockerBaseImagePythonVersion() + " && pip install -r /root/" + environmentFile);
                    }
                }
                writer.write(" && " + getCleanupCommand() + " && " + anaconda_dir + "/bin/conda list -n " + settings.getCurrentCondaEnvironment());
            }
        }
        Project project = projectFacade.findById(cc.getProjectId().getId()).orElseThrow(() -> new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "projectId: " + cc.getProjectId().getId()));
        String initialDockerImage = projectUtils.getInitialDockerImageName(project);
        LOG.log(Level.FINEST, "project-initialDockerImage:" + initialDockerImage);
        ProcessDescriptor processDescriptor = new ProcessDescriptor.Builder().addCommand("/usr/bin/sudo").addCommand(prog).addCommand("create").addCommand(dockerFile.getAbsolutePath()).addCommand(projectUtils.getRegistryURL() + "/" + initialDockerImage).redirectErrorStream(true).setCurrentWorkingDirectory(baseDir).setWaitTimeout(1, TimeUnit.HOURS).build();
        ProcessResult processResult = osProcessExecutor.execute(processDescriptor);
        if (processResult.getExitCode() != 0) {
            String errorMsg = "Could not create the docker image. Exit code: " + processResult.getExitCode() + " out: " + processResult.getStdout() + "\n err: " + processResult.getStderr() + "||\n";
            throw new IOException(errorMsg);
        } else {
            project = projectFacade.findById(cc.getProjectId().getId()).orElseThrow(() -> new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "projectId: " + cc.getProjectId().getId()));
            project.setDockerImage(initialDockerImage);
            project = projectFacade.update(project);
            projectFacade.flushEm();
            setPipConflicts(project);
            environmentController.updateInstalledDependencies(project);
            exportEnvironment(project, cc.getUserId(), Utils.getProjectPath(project.getName()) + Settings.PROJECT_PYTHON_ENVIRONMENT_FILE);
        }
    } catch (ServiceException | ProjectException | PythonException e) {
        LOG.log(Level.SEVERE, "Failed to persist python deps", e);
    } finally {
        FileUtils.deleteDirectory(baseDir);
    }
}