Examples with ProcessResult io.hops.hopsworks.common.util.ProcessResult used on opensource projects

Example 21 with ProcessResult

use of io.hops.hopsworks.common.util.ProcessResult in project hopsworks by logicalclocks.

the class LibraryInstaller method uninstallLibrary.

private void uninstallLibrary(CondaCommands cc) throws IOException, ServiceDiscoveryException, ProjectException, ServiceException, PythonException {
    File baseDir = new File("/tmp/docker/" + cc.getProjectId().getName());
    baseDir.mkdirs();
    try {
        File dockerFile = new File(baseDir, "dockerFile_" + cc.getProjectId().getName());
        File home = new File(System.getProperty("user.home"));
        FileUtils.copyFileToDirectory(new File(home, ".condarc"), baseDir);
        FileUtils.copyDirectoryToDirectory(new File(home, ".pip"), baseDir);
        try (BufferedWriter writer = new BufferedWriter(new FileWriter(dockerFile))) {
            writer.write("# syntax=docker/dockerfile:experimental");
            writer.newLine();
            writer.write("FROM " + projectUtils.getFullDockerImageName(cc.getProjectId(), false) + "\n");
            writer.newLine();
            writer.write("RUN --mount=type=bind,source=.condarc,target=/root/.condarc" + " --mount=type=bind,source=.pip,target=/root/.pip ");
            switch(cc.getInstallType()) {
                case CONDA:
                    writer.write(anaconda_dir + "/bin/conda remove -y -n " + settings.getCurrentCondaEnvironment() + " " + cc.getLib() + " || true\n");
                    break;
                case PIP:
                    writer.write(anaconda_project_dir + "/bin/pip uninstall -y " + cc.getLib() + " || true\n");
                    break;
                case ENVIRONMENT:
                default:
                    throw new UnsupportedOperationException("install type unknown: " + cc.getInstallType());
            }
        }
        Project project = projectFacade.findById(cc.getProjectId().getId()).orElseThrow(() -> new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "projectId: " + cc.getProjectId().getId()));
        String nextDockerImageName = getNextDockerImageName(project);
        ProcessDescriptor processDescriptor = new ProcessDescriptor.Builder().addCommand("/usr/bin/sudo").addCommand(prog).addCommand("create").addCommand(dockerFile.getAbsolutePath()).addCommand(projectUtils.getRegistryURL() + "/" + nextDockerImageName).redirectErrorStream(true).setCurrentWorkingDirectory(baseDir).setWaitTimeout(30, TimeUnit.MINUTES).build();
        ProcessResult processResult = osProcessExecutor.execute(processDescriptor);
        if (processResult.getExitCode() != 0) {
            String errorMsg = "Could not create the docker image. Exit code: " + processResult.getExitCode() + " out: " + processResult.getStdout() + "\n err: " + processResult.getStderr() + "||\n";
            throw new IOException(errorMsg);
        } else {
            project = projectFacade.findById(cc.getProjectId().getId()).orElseThrow(() -> new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "projectId: " + cc.getProjectId().getId()));
            project.setDockerImage(nextDockerImageName);
            project = projectFacade.update(project);
            projectFacade.flushEm();
            setPipConflicts(project);
            environmentController.updateInstalledDependencies(project);
            exportEnvironment(project, cc.getUserId(), Utils.getProjectPath(project.getName()) + Settings.PROJECT_PYTHON_ENVIRONMENT_FILE);
        }
    } finally {
        FileUtils.deleteDirectory(baseDir);
    }
}

Example 22 with ProcessResult

use of io.hops.hopsworks.common.util.ProcessResult in project hopsworks by logicalclocks.

the class LibraryInstaller method installLibrary.

private void installLibrary(CondaCommands cc) throws IOException, ServiceException, ServiceDiscoveryException, ProjectException, UserException, PythonException {
    File baseDir = new File("/tmp/docker/" + cc.getProjectId().getName());
    baseDir.mkdirs();
    Project project = projectFacade.findById(cc.getProjectId().getId()).orElseThrow(() -> new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "projectId: " + cc.getProjectId().getId()));
    ArrayList<String> dockerBuildOpts = new ArrayList<>();
    dockerBuildOpts.add(DOCKER_HOST_NETWORK_OPT);
    try {
        File home = new File(System.getProperty("user.home"));
        File condarc = new File(home, ".condarc");
        File pip = new File(home, ".pip");
        FileUtils.copyFileToDirectory(condarc, baseDir);
        FileUtils.copyDirectoryToDirectory(pip, baseDir);
        File dockerFile = new File(baseDir, "dockerFile_" + cc.getProjectId().getName());
        String apiToken = null;
        try (BufferedWriter writer = new BufferedWriter(new FileWriter(dockerFile))) {
            writer.write("# syntax=docker/dockerfile:experimental");
            writer.newLine();
            writer.write("FROM " + projectUtils.getFullDockerImageName(project, false));
            writer.newLine();
            writer.write("RUN --mount=type=bind,source=.condarc,target=/root/.condarc" + " --mount=type=bind,source=.pip,target=/root/.pip ");
            switch(cc.getInstallType()) {
                case CONDA:
                    String condaLib;
                    if (cc.getVersion().equals(Settings.UNKNOWN_LIBRARY_VERSION)) {
                        condaLib = cc.getLib();
                        dockerBuildOpts.add(DOCKER_NO_CACHE_OPT);
                    } else {
                        condaLib = cc.getLib() + "=" + cc.getVersion();
                    }
                    writer.write(anaconda_dir + "/bin/conda install -y -n " + settings.getCurrentCondaEnvironment() + " -c " + cc.getChannelUrl() + " " + condaLib);
                    break;
                case PIP:
                    String pipLib;
                    if (cc.getVersion().equals(Settings.UNKNOWN_LIBRARY_VERSION)) {
                        pipLib = cc.getLib();
                        dockerBuildOpts.add(DOCKER_NO_CACHE_OPT);
                    } else {
                        pipLib = cc.getLib() + "==" + cc.getVersion();
                    }
                    writer.write(anaconda_project_dir + "/bin/pip install --upgrade " + pipLib);
                    break;
                case EGG:
                    String eggName = cc.getLib();
                    String localEggPath = baseDir + File.separator + eggName;
                    copyCondaArtifactToLocal(cc.getArg(), localEggPath);
                    writer.write("--mount=type=bind,source=" + eggName + ",target=/root/" + eggName + " ");
                    writer.write(anaconda_project_dir + "/bin/easy_install --upgrade /root/" + eggName);
                    break;
                case WHEEL:
                    String wheelName = cc.getLib();
                    String localWheelPath = baseDir + File.separator + wheelName;
                    copyCondaArtifactToLocal(cc.getArg(), localWheelPath);
                    writer.write("--mount=type=bind,source=" + wheelName + ",target=/root/" + wheelName + " ");
                    writer.write(anaconda_project_dir + "/bin/pip install --upgrade /root/" + wheelName);
                    break;
                case REQUIREMENTS_TXT:
                    String requirementsName = cc.getLib();
                    String localRequirementsName = baseDir + File.separator + requirementsName;
                    copyCondaArtifactToLocal(cc.getArg(), localRequirementsName);
                    writer.write("--mount=type=bind,source=" + requirementsName + ",target=/root/" + requirementsName + " ");
                    writer.write(anaconda_project_dir + "/bin/pip install -r /root/" + requirementsName);
                    break;
                case ENVIRONMENT_YAML:
                    String environmentsName = cc.getLib();
                    String localEnvironmentsName = baseDir + File.separator + environmentsName;
                    copyCondaArtifactToLocal(cc.getArg(), localEnvironmentsName);
                    writer.write("--mount=type=bind,source=" + environmentsName + ",target=/root/" + environmentsName + " ");
                    writer.write(anaconda_dir + "/bin/conda env update -f /root/" + environmentsName + " -n " + settings.getCurrentCondaEnvironment());
                    break;
                case GIT:
                    if (cc.getGitBackend() != null && cc.getGitApiKeyName() != null) {
                        apiToken = this.secretsController.get(cc.getUserId(), cc.getGitApiKeyName()).getPlaintext();
                        URL repoUrl = new URL(cc.getArg());
                        if (cc.getGitBackend().equals(GitBackend.GITHUB)) {
                            writer.write(anaconda_project_dir + "/bin/pip install --upgrade 'git+https://" + apiToken + ":x-oauth-basic@" + repoUrl.getHost() + repoUrl.getPath() + "'");
                        } else if (cc.getGitBackend().equals(GitBackend.GITLAB)) {
                            writer.write(anaconda_project_dir + "/bin/pip install --upgrade 'git+https://oauth2:" + apiToken + "@" + repoUrl.getHost() + repoUrl.getPath() + "'");
                        }
                    } else {
                        writer.write(anaconda_project_dir + "/bin/pip install --upgrade 'git+" + cc.getArg() + "'");
                    }
                    dockerBuildOpts.add(DOCKER_NO_CACHE_OPT);
                    break;
                case ENVIRONMENT:
                default:
                    throw new UnsupportedOperationException("install type unknown: " + cc.getInstallType());
            }
            // Installing faulty libraries like broken .egg files can cause the list operation to fail
            // As we find library names and versions using that command we need to make sure it does not break
            writer.write(" && " + getCleanupCommand() + " && " + anaconda_dir + "/bin/conda list -n " + settings.getCurrentCondaEnvironment());
        }
        String nextDockerImageName = getNextDockerImageName(project);
        LOG.log(Level.FINEST, "project-nextDockerImageName:" + nextDockerImageName);
        ProcessDescriptor processDescriptor = new ProcessDescriptor.Builder().addCommand("/usr/bin/sudo").addCommand(prog).addCommand("create").addCommand(dockerFile.getAbsolutePath()).addCommand(projectUtils.getRegistryURL() + "/" + nextDockerImageName).addCommand("'" + String.join(" ", dockerBuildOpts) + "'").redirectErrorStream(true).setCurrentWorkingDirectory(baseDir).setWaitTimeout(1, TimeUnit.HOURS).build();
        ProcessResult processResult = osProcessExecutor.execute(processDescriptor);
        if (processResult.getExitCode() != 0) {
            // Avoid leeking the apitoken in the error logs by replacing it with the name
            if (cc.getInstallType().equals(CondaInstallType.GIT) && !Strings.isNullOrEmpty(apiToken)) {
                String errorMsg = "Could not create the docker image. Exit code: " + processResult.getExitCode();
                String stdout = processResult.getStdout();
                if (stdout != null) {
                    errorMsg = errorMsg + " out: " + stdout.replaceAll(apiToken, cc.getGitApiKeyName() + "_token");
                }
                String stderr = processResult.getStderr();
                if (stderr != null) {
                    errorMsg = errorMsg + "\n err: " + stderr.replaceAll(apiToken, cc.getGitApiKeyName() + "_token");
                }
                errorMsg = errorMsg + "||\n";
                throw new IOException(errorMsg);
            } else {
                String errorMsg = "Could not create the docker image. Exit code: " + processResult.getExitCode() + " out: " + processResult.getStdout() + "\n err: " + processResult.getStderr() + "||\n";
                throw new IOException(errorMsg);
            }
        } else {
            project = projectFacade.findById(cc.getProjectId().getId()).orElseThrow(() -> new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "projectId: " + cc.getProjectId().getId()));
            project.setDockerImage(nextDockerImageName);
            project = projectFacade.update(project);
            projectFacade.flushEm();
            setPipConflicts(project);
            environmentController.updateInstalledDependencies(project);
            exportEnvironment(project, cc.getUserId(), Utils.getProjectPath(project.getName()) + Settings.PROJECT_PYTHON_ENVIRONMENT_FILE);
        }
    } finally {
        FileUtils.deleteDirectory(baseDir);
    }
}

Example 23 with ProcessResult

use of io.hops.hopsworks.common.util.ProcessResult in project hopsworks by logicalclocks.

the class JupyterController method convertIPythonNotebook.

@TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED)
public String convertIPythonNotebook(String hdfsUsername, String notebookPath, Project project, String pyPath, NotebookConversion notebookConversion) throws ServiceException {
    File baseDir = new File(settings.getStagingDir() + settings.CONVERSION_DIR);
    if (!baseDir.exists()) {
        baseDir.mkdir();
    }
    File conversionDir = new File(baseDir, DigestUtils.sha256Hex(Integer.toString(ThreadLocalRandom.current().nextInt())));
    conversionDir.mkdir();
    HdfsUsers user = hdfsUsersFacade.findByName(hdfsUsername);
    try {
        String prog = settings.getSudoersDir() + "/convert-ipython-notebook.sh";
        ProcessDescriptor processDescriptor = new ProcessDescriptor.Builder().addCommand("/usr/bin/sudo").addCommand(prog).addCommand(notebookPath).addCommand(hdfsUsername).addCommand(settings.getAnacondaProjectDir()).addCommand(pyPath).addCommand(conversionDir.getAbsolutePath()).addCommand(notebookConversion.name()).addCommand(projectUtils.getFullDockerImageName(project, true)).setWaitTimeout(60l, // on a TLS VM the timeout needs to be greater than 20s
        TimeUnit.SECONDS).redirectErrorStream(true).build();
        LOGGER.log(Level.FINE, processDescriptor.toString());
        certificateMaterializer.materializeCertificatesLocalCustomDir(user.getUsername(), project.getName(), conversionDir.getAbsolutePath());
        ProcessResult processResult = osProcessExecutor.execute(processDescriptor);
        if (!processResult.processExited() || processResult.getExitCode() != 0) {
            LOGGER.log(Level.WARNING, "error code: " + processResult.getExitCode(), "Failed to convert " + notebookPath + "\nstderr: " + processResult.getStderr() + "\nstdout: " + processResult.getStdout());
            throw new ServiceException(RESTCodes.ServiceErrorCode.IPYTHON_CONVERT_ERROR, Level.SEVERE, "error code: " + processResult.getExitCode(), "Failed to convert " + notebookPath + "\nstderr: " + processResult.getStderr() + "\nstdout: " + processResult.getStdout());
        }
        String stdOut = processResult.getStdout();
        if (!Strings.isNullOrEmpty(stdOut) && notebookConversion.equals(NotebookConversion.HTML)) {
            StringBuilder renderedNotebookSB = new StringBuilder(stdOut);
            int startIndex = renderedNotebookSB.indexOf("<html>");
            int stopIndex = renderedNotebookSB.length();
            return renderedNotebookSB.substring(startIndex, stopIndex);
        }
        return null;
    } catch (IOException | ServiceDiscoveryException ex) {
        throw new ServiceException(RESTCodes.ServiceErrorCode.IPYTHON_CONVERT_ERROR, Level.SEVERE, null, ex.getMessage(), ex);
    } finally {
        certificateMaterializer.removeCertificatesLocalCustomDir(user.getUsername(), project.getName(), conversionDir.getAbsolutePath());
    }
}

Example 24 with ProcessResult

use of io.hops.hopsworks.common.util.ProcessResult in project hopsworks by logicalclocks.

the class LocalHostJupyterProcessMgr method executeJupyterCommand.

private int executeJupyterCommand(String... args) {
    if (args == null || args.length == 0) {
        return -99;
    }
    int exitValue;
    Integer id = 1;
    String prog = settings.getSudoersDir() + "/jupyter.sh";
    ProcessDescriptor.Builder pdBuilder = new ProcessDescriptor.Builder().addCommand("/usr/bin/sudo").addCommand(prog);
    for (String arg : args) {
        pdBuilder.addCommand(arg);
    }
    pdBuilder.setWaitTimeout(20L, TimeUnit.SECONDS);
    if (!LOGGER.isLoggable(Level.FINE)) {
        pdBuilder.ignoreOutErrStreams(true);
    }
    try {
        ProcessResult processResult = osProcessExecutor.execute(pdBuilder.build());
        LOGGER.log(Level.FINE, processResult.getStdout());
        exitValue = processResult.getExitCode();
    } catch (IOException ex) {
        LOGGER.log(Level.SEVERE, "Problem checking if Jupyter Notebook server is running: {0}", ex);
        exitValue = -2;
    }
    return exitValue;
}