Examples with PythonDep io.hops.hopsworks.persistence.entity.python.PythonDep used on opensource projects

Example 1 with PythonDep

use of io.hops.hopsworks.persistence.entity.python.PythonDep in project hopsworks by logicalclocks.

the class LibraryResource method install.

@ApiOperation(value = "Install a python library in the environment")
@POST
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@Path("{library}")
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.PYTHON }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response install(LibrarySpecification librarySpecification, @PathParam("library") String library, @Context UriInfo uriInfo, @Context HttpServletRequest req, @Context SecurityContext sc) throws ServiceException, GenericException, PythonException, DatasetException {
    Users user = jwtHelper.getUserPrincipal(req);
    environmentController.checkCondaEnabled(project, pythonVersion, true);
    PackageSource packageSource = librarySpecification.getPackageSource();
    if (packageSource == null) {
        throw new PythonException(RESTCodes.PythonErrorCode.INSTALL_TYPE_NOT_SUPPORTED, Level.FINE);
    }
    switch(packageSource) {
        case PIP:
            validateLibrary(librarySpecification, library);
            librarySpecification.setChannelUrl("pypi");
            break;
        case CONDA:
            validateLibrary(librarySpecification, library);
            break;
        case EGG:
        case WHEEL:
        case REQUIREMENTS_TXT:
        case ENVIRONMENT_YAML:
            validateBundledDependency(user, librarySpecification);
            break;
        case GIT:
            validateGitURL(librarySpecification.getDependencyUrl());
            break;
        default:
            throw new PythonException(RESTCodes.PythonErrorCode.INSTALL_TYPE_NOT_SUPPORTED, Level.FINE);
    }
    environmentController.checkCondaEnvExists(project, user);
    PythonDep dep = libraryController.installLibrary(project, user, CondaInstallType.valueOf(packageSource.name().toUpperCase()), librarySpecification.getChannelUrl(), library, librarySpecification.getVersion(), librarySpecification.getDependencyUrl(), librarySpecification.getGitBackend(), librarySpecification.getGitApiKey());
    ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.LIBRARIES);
    LibraryDTO libraryDTO = librariesBuilder.build(uriInfo, resourceRequest, dep, project);
    return Response.created(libraryDTO.getHref()).entity(libraryDTO).build();
}

Example 2 with PythonDep

use of io.hops.hopsworks.persistence.entity.python.PythonDep in project hopsworks by logicalclocks.

the class LibraryResource method getByName.

@ApiOperation(value = "Get a python library installed in this environment", response = LibraryDTO.class)
@GET
@Path("{library}")
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.PYTHON }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response getByName(@PathParam("library") String library, @BeanParam LibraryExpansionBeanParam expansions, @Context UriInfo uriInfo, @Context SecurityContext sc) throws PythonException {
    validatePattern(library);
    environmentController.checkCondaEnabled(project, pythonVersion, true);
    PythonDep dep = libraryController.getPythonDep(library, project);
    ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.LIBRARIES);
    if (expansions != null) {
        resourceRequest.setExpansions(expansions.getResources());
    }
    LibraryDTO libraryDTO = librariesBuilder.buildItem(uriInfo, resourceRequest, dep, project);
    return Response.ok().entity(libraryDTO).build();
}

Example 3 with PythonDep

use of io.hops.hopsworks.persistence.entity.python.PythonDep in project hopsworks by logicalclocks.

the class CommandsController method condaOp.

public PythonDep condaOp(CondaOp op, Users user, CondaInstallType installType, Project proj, String channelUrl, String lib, String version, String arg, GitBackend gitBackend, String apiKeyName) throws GenericException, ServiceException {
    if (Strings.isNullOrEmpty(version) && CondaOp.isLibraryOp(op)) {
        version = Settings.UNKNOWN_LIBRARY_VERSION;
    }
    // If there is an already ongoing command to uninstall the same library, allow the queuing of a new install op
    List<CondaStatus> statuses = new ArrayList<>();
    statuses.add(CondaStatus.NEW);
    statuses.add(CondaStatus.ONGOING);
    List<CondaCommands> uninstallCommands = condaCommandFacade.findByStatusAndCondaOpAndProject(statuses, CondaOp.UNINSTALL, proj);
    // Get current uninstall operations for this project
    List<CondaCommands> ongoingUninstall = uninstallCommands.stream().filter(c -> c.getLib().equalsIgnoreCase(lib) && c.getInstallType().equals(installType)).collect(Collectors.toList());
    // Get currently installed library with the same name and package manager if it exists
    Optional<PythonDep> installedDep = proj.getPythonDepCollection().stream().filter(d -> d.getDependency().equalsIgnoreCase(lib) && d.getInstallType().equals(installType)).findFirst();
    if (op == CondaOp.INSTALL && installedDep.isPresent() && ongoingUninstall.isEmpty()) {
        throw new ServiceException(RESTCodes.ServiceErrorCode.ANACONDA_DEP_INSTALL_FORBIDDEN, Level.FINE, "dep: " + lib);
    }
    PythonDep dep;
    try {
        // 1. test if anacondaRepoUrl exists. If not, add it.
        AnacondaRepo repo = libraryFacade.getRepo(channelUrl, true);
        // 2. Test if pythonDep exists. If not, add it.
        dep = libraryFacade.getOrCreateDep(repo, installType, lib, version, true, false);
        Collection<PythonDep> depsInProj = proj.getPythonDepCollection();
        // 3. Add the python library to the join table for the project
        if (op == CondaOp.INSTALL) {
            // if upgrade
            depsInProj.remove(dep);
            depsInProj.add(dep);
        }
        proj.setPythonDepCollection(depsInProj);
        projectFacade.update(proj);
        CondaCommands cc = new CondaCommands(user, op, CondaStatus.NEW, installType, proj, lib, version, channelUrl, new Date(), arg, null, false, gitBackend, apiKeyName);
        condaCommandFacade.save(cc);
    } catch (Exception ex) {
        throw new GenericException(RESTCodes.GenericErrorCode.UNKNOWN_ERROR, Level.SEVERE, "condaOp failed", ex.getMessage(), ex);
    }
    return dep;
}

Example 4 with PythonDep

use of io.hops.hopsworks.persistence.entity.python.PythonDep in project hopsworks by logicalclocks.

the class EnvironmentController method updateInstalledDependencies.

public Project updateInstalledDependencies(Project project) throws ServiceException, IOException {
    try {
        String condaListOutput = libraryController.condaList(projectUtils.getFullDockerImageName(project, false));
        Collection<PythonDep> projectDeps = libraryController.parseCondaList(condaListOutput);
        projectDeps = libraryController.persistAndMarkImmutable(projectDeps);
        project = libraryController.syncProjectPythonDepsWithEnv(project, projectDeps);
        project = libraryController.addOngoingOperations(project);
        return project;
    } catch (ServiceDiscoveryException e) {
        throw new ServiceException(RESTCodes.ServiceErrorCode.SERVICE_DISCOVERY_ERROR, Level.SEVERE, null, e.getMessage(), e);
    }
}

Example 5 with PythonDep

use of io.hops.hopsworks.persistence.entity.python.PythonDep in project hopsworks by logicalclocks.

the class LibraryController method addOngoingOperations.

public Project addOngoingOperations(Project project) throws ServiceException {
    Collection<CondaCommands> commands = project.getCondaCommandsCollection();
    for (CondaCommands condaCommand : commands) {
        if (condaCommand.getInstallType().equals(CondaInstallType.ENVIRONMENT))
            continue;
        PythonDep pythonDep = new PythonDep();
        pythonDep.setDependency(condaCommand.getLib());
        pythonDep.setInstallType(condaCommand.getInstallType());
        pythonDep.setPreinstalled(false);
        AnacondaRepo repo = libraryFacade.getRepo(condaCommand.getChannelUrl(), false);
        pythonDep.setRepoUrl(repo);
        pythonDep.setVersion(condaCommand.getVersion());
        pythonDep = libraryFacade.getOrCreateDep(pythonDep);
        if (!project.getPythonDepCollection().contains(pythonDep)) {
            project.getPythonDepCollection().add(pythonDep);
        }
    }
    return projectFacade.update(project);
}