Examples with CondaCommands io.hops.hopsworks.persistence.entity.python.CondaCommands used on opensource projects

Example 1 with CondaCommands

use of io.hops.hopsworks.persistence.entity.python.CondaCommands in project hopsworks by logicalclocks.

the class CondaCommandFacade method findAllLibCmdByProject.

public CollectionInfo findAllLibCmdByProject(Integer offset, Integer limit, Set<? extends AbstractFacade.FilterBy> filter, Set<? extends AbstractFacade.SortBy> sort, Project project, String libName) {
    String queryStr = buildQuery("SELECT c FROM CondaCommands c ", filter, sort, "c.lib = :lib AND c.installType <> :installType AND c.projectId = :project ");
    String queryCountStr = buildQuery("SELECT COUNT(c.id) FROM CondaCommands c ", filter, sort, "c.lib = :lib AND c.installType <> :installType AND c.projectId = :project ");
    Query query = em.createQuery(queryStr, CondaCommands.class).setParameter("lib", libName).setParameter("installType", CondaInstallType.ENVIRONMENT).setParameter("project", project);
    Query queryCount = em.createQuery(queryCountStr, CondaCommands.class).setParameter("lib", libName).setParameter("installType", CondaInstallType.ENVIRONMENT).setParameter("project", project);
    return findAll(offset, limit, filter, query, queryCount);
}

Example 2 with CondaCommands

use of io.hops.hopsworks.persistence.entity.python.CondaCommands in project hopsworks by logicalclocks.

the class CommandsController method retryFailedCondaEnvOps.

public void retryFailedCondaEnvOps(Project project) {
    List<CondaCommands> commands = condaCommandFacade.getFailedEnvCommandsForProject(project);
    for (CondaCommands cc : commands) {
        cc.setStatus(CondaStatus.NEW);
        condaCommandFacade.update(cc);
    }
}

Example 3 with CondaCommands

use of io.hops.hopsworks.persistence.entity.python.CondaCommands in project hopsworks by logicalclocks.

the class CommandsController method retryFailedCondaLibraryOps.

public void retryFailedCondaLibraryOps(Project project, String library) {
    List<CondaCommands> commands = condaCommandFacade.getFailedCommandsForProjectAndLib(project, library);
    for (CondaCommands cc : commands) {
        cc.setStatus(CondaStatus.NEW);
        condaCommandFacade.update(cc);
    }
}

Example 4 with CondaCommands

use of io.hops.hopsworks.persistence.entity.python.CondaCommands in project hopsworks by logicalclocks.

the class CommandsController method condaOp.

public PythonDep condaOp(CondaOp op, Users user, CondaInstallType installType, Project proj, String channelUrl, String lib, String version, String arg, GitBackend gitBackend, String apiKeyName) throws GenericException, ServiceException {
    if (Strings.isNullOrEmpty(version) && CondaOp.isLibraryOp(op)) {
        version = Settings.UNKNOWN_LIBRARY_VERSION;
    }
    // If there is an already ongoing command to uninstall the same library, allow the queuing of a new install op
    List<CondaStatus> statuses = new ArrayList<>();
    statuses.add(CondaStatus.NEW);
    statuses.add(CondaStatus.ONGOING);
    List<CondaCommands> uninstallCommands = condaCommandFacade.findByStatusAndCondaOpAndProject(statuses, CondaOp.UNINSTALL, proj);
    // Get current uninstall operations for this project
    List<CondaCommands> ongoingUninstall = uninstallCommands.stream().filter(c -> c.getLib().equalsIgnoreCase(lib) && c.getInstallType().equals(installType)).collect(Collectors.toList());
    // Get currently installed library with the same name and package manager if it exists
    Optional<PythonDep> installedDep = proj.getPythonDepCollection().stream().filter(d -> d.getDependency().equalsIgnoreCase(lib) && d.getInstallType().equals(installType)).findFirst();
    if (op == CondaOp.INSTALL && installedDep.isPresent() && ongoingUninstall.isEmpty()) {
        throw new ServiceException(RESTCodes.ServiceErrorCode.ANACONDA_DEP_INSTALL_FORBIDDEN, Level.FINE, "dep: " + lib);
    }
    PythonDep dep;
    try {
        // 1. test if anacondaRepoUrl exists. If not, add it.
        AnacondaRepo repo = libraryFacade.getRepo(channelUrl, true);
        // 2. Test if pythonDep exists. If not, add it.
        dep = libraryFacade.getOrCreateDep(repo, installType, lib, version, true, false);
        Collection<PythonDep> depsInProj = proj.getPythonDepCollection();
        // 3. Add the python library to the join table for the project
        if (op == CondaOp.INSTALL) {
            // if upgrade
            depsInProj.remove(dep);
            depsInProj.add(dep);
        }
        proj.setPythonDepCollection(depsInProj);
        projectFacade.update(proj);
        CondaCommands cc = new CondaCommands(user, op, CondaStatus.NEW, installType, proj, lib, version, channelUrl, new Date(), arg, null, false, gitBackend, apiKeyName);
        condaCommandFacade.save(cc);
    } catch (Exception ex) {
        throw new GenericException(RESTCodes.GenericErrorCode.UNKNOWN_ERROR, Level.SEVERE, "condaOp failed", ex.getMessage(), ex);
    }
    return dep;
}

Example 5 with CondaCommands

use of io.hops.hopsworks.persistence.entity.python.CondaCommands in project hopsworks by logicalclocks.

the class DockerRegistryMngrImpl method gc.

@Override
public void gc() throws IOException, ServiceException, ProjectException {
    // 1. Get all conda commands of type REMOVE. Should be only 1 REMOVE per project
    final List<CondaCommands> condaCommandsRemove = condaCommandFacade.findByStatusAndCondaOp(CondaStatus.NEW, CondaOp.REMOVE);
    LOG.log(Level.FINE, "condaCommandsRemove: " + condaCommandsRemove);
    try {
        for (CondaCommands cc : condaCommandsRemove) {
            // We do not want to remove the base image! Get arguments from command as project may have already been deleted.
            String projectDockerImage = cc.getArg();
            String projectDockerRepoName = projectUtils.getProjectDockerRepoName(projectDockerImage);
            if (!projectUtils.dockerImageIsPreinstalled(projectDockerImage)) {
                try {
                    // 1. Get and delete all the tags for each repository(project)
                    List<String> projectTags = deleteProjectImagesOnRegistry(projectDockerImage);
                    for (String tag : projectTags) {
                        // Issue system command (kagent) to remove docker image from each host's docker daemon
                        dockerImagesGC(projectUtils.getRegistryURL() + "/" + projectDockerRepoName + ":" + tag);
                    }
                } catch (Exception ex) {
                    LOG.log(Level.WARNING, "Could not complete docker registry cleanup for: " + cc, ex);
                    try {
                        commandsController.updateCondaCommandStatus(cc.getId(), CondaStatus.FAILED, cc.getArg(), cc.getOp(), "Could not complete docker registry cleanup: " + ex.getMessage());
                    } catch (ServiceException | ProjectException e) {
                        LOG.log(Level.WARNING, "Could not change conda command status to NEW.", e);
                    }
                }
            }
            commandsController.updateCondaCommandStatus(cc.getId(), CondaStatus.SUCCESS, cc.getArg(), cc.getOp());
        }
    } finally {
        // Run docker gc in cli
        if (!condaCommandsRemove.isEmpty()) {
            runRegistryGC();
        }
    }
}