Examples with ServiceException io.hops.hopsworks.exceptions.ServiceException used on opensource projects

Example 21 with ServiceException

use of io.hops.hopsworks.exceptions.ServiceException in project hopsworks by logicalclocks.

the class OnlineFeaturegroupController method setupOnlineFeatureGroup.

public void setupOnlineFeatureGroup(Featurestore featureStore, Integer featureGroupId, String featureGroupName, Integer featureGroupVersion, List<FeatureGroupFeatureDTO> features, Project project, Users user) throws KafkaException, SchemaException, ProjectException, UserException, FeaturestoreException, SQLException, IOException, HopsSecurityException, ServiceException {
    // check if onlinefs user is part of project
    if (project.getProjectTeamCollection().stream().noneMatch(pt -> pt.getUser().getUsername().equals(OnlineFeaturestoreController.ONLINEFS_USERNAME))) {
        try {
            // wait for the future
            projectController.addOnlineFsUser(project).get();
        } catch (InterruptedException | ExecutionException e) {
            throw new ServiceException(RESTCodes.ServiceErrorCode.SERVICE_GENERIC_ERROR, Level.SEVERE, "failed to add onlinefs user to project: " + project.getName(), e.getMessage(), e);
        }
    }
    String featureGroupEntityName = Utils.getFeatureStoreEntityName(featureGroupName, featureGroupVersion);
    createMySQLTable(featureStore, featureGroupEntityName, features, project, user);
    String avroSchema = avroSchemaConstructorController.constructSchema(featureGroupEntityName, Utils.getFeaturestoreName(project), features);
    schemasController.validateSchema(project, avroSchema);
    createOnlineKafkaTopic(project, featureGroupId, featureGroupEntityName, avroSchema);
}

Example 22 with ServiceException

use of io.hops.hopsworks.exceptions.ServiceException in project hopsworks by logicalclocks.

the class CommandsController method condaOp.

public PythonDep condaOp(CondaOp op, Users user, CondaInstallType installType, Project proj, String channelUrl, String lib, String version, String arg, GitBackend gitBackend, String apiKeyName) throws GenericException, ServiceException {
    if (Strings.isNullOrEmpty(version) && CondaOp.isLibraryOp(op)) {
        version = Settings.UNKNOWN_LIBRARY_VERSION;
    }
    // If there is an already ongoing command to uninstall the same library, allow the queuing of a new install op
    List<CondaStatus> statuses = new ArrayList<>();
    statuses.add(CondaStatus.NEW);
    statuses.add(CondaStatus.ONGOING);
    List<CondaCommands> uninstallCommands = condaCommandFacade.findByStatusAndCondaOpAndProject(statuses, CondaOp.UNINSTALL, proj);
    // Get current uninstall operations for this project
    List<CondaCommands> ongoingUninstall = uninstallCommands.stream().filter(c -> c.getLib().equalsIgnoreCase(lib) && c.getInstallType().equals(installType)).collect(Collectors.toList());
    // Get currently installed library with the same name and package manager if it exists
    Optional<PythonDep> installedDep = proj.getPythonDepCollection().stream().filter(d -> d.getDependency().equalsIgnoreCase(lib) && d.getInstallType().equals(installType)).findFirst();
    if (op == CondaOp.INSTALL && installedDep.isPresent() && ongoingUninstall.isEmpty()) {
        throw new ServiceException(RESTCodes.ServiceErrorCode.ANACONDA_DEP_INSTALL_FORBIDDEN, Level.FINE, "dep: " + lib);
    }
    PythonDep dep;
    try {
        // 1. test if anacondaRepoUrl exists. If not, add it.
        AnacondaRepo repo = libraryFacade.getRepo(channelUrl, true);
        // 2. Test if pythonDep exists. If not, add it.
        dep = libraryFacade.getOrCreateDep(repo, installType, lib, version, true, false);
        Collection<PythonDep> depsInProj = proj.getPythonDepCollection();
        // 3. Add the python library to the join table for the project
        if (op == CondaOp.INSTALL) {
            // if upgrade
            depsInProj.remove(dep);
            depsInProj.add(dep);
        }
        proj.setPythonDepCollection(depsInProj);
        projectFacade.update(proj);
        CondaCommands cc = new CondaCommands(user, op, CondaStatus.NEW, installType, proj, lib, version, channelUrl, new Date(), arg, null, false, gitBackend, apiKeyName);
        condaCommandFacade.save(cc);
    } catch (Exception ex) {
        throw new GenericException(RESTCodes.GenericErrorCode.UNKNOWN_ERROR, Level.SEVERE, "condaOp failed", ex.getMessage(), ex);
    }
    return dep;
}

Example 23 with ServiceException

use of io.hops.hopsworks.exceptions.ServiceException in project hopsworks by logicalclocks.

the class DockerRegistryMngrImpl method gc.

@Override
public void gc() throws IOException, ServiceException, ProjectException {
    // 1. Get all conda commands of type REMOVE. Should be only 1 REMOVE per project
    final List<CondaCommands> condaCommandsRemove = condaCommandFacade.findByStatusAndCondaOp(CondaStatus.NEW, CondaOp.REMOVE);
    LOG.log(Level.FINE, "condaCommandsRemove: " + condaCommandsRemove);
    try {
        for (CondaCommands cc : condaCommandsRemove) {
            // We do not want to remove the base image! Get arguments from command as project may have already been deleted.
            String projectDockerImage = cc.getArg();
            String projectDockerRepoName = projectUtils.getProjectDockerRepoName(projectDockerImage);
            if (!projectUtils.dockerImageIsPreinstalled(projectDockerImage)) {
                try {
                    // 1. Get and delete all the tags for each repository(project)
                    List<String> projectTags = deleteProjectImagesOnRegistry(projectDockerImage);
                    for (String tag : projectTags) {
                        // Issue system command (kagent) to remove docker image from each host's docker daemon
                        dockerImagesGC(projectUtils.getRegistryURL() + "/" + projectDockerRepoName + ":" + tag);
                    }
                } catch (Exception ex) {
                    LOG.log(Level.WARNING, "Could not complete docker registry cleanup for: " + cc, ex);
                    try {
                        commandsController.updateCondaCommandStatus(cc.getId(), CondaStatus.FAILED, cc.getArg(), cc.getOp(), "Could not complete docker registry cleanup: " + ex.getMessage());
                    } catch (ServiceException | ProjectException e) {
                        LOG.log(Level.WARNING, "Could not change conda command status to NEW.", e);
                    }
                }
            }
            commandsController.updateCondaCommandStatus(cc.getId(), CondaStatus.SUCCESS, cc.getArg(), cc.getOp());
        }
    } finally {
        // Run docker gc in cli
        if (!condaCommandsRemove.isEmpty()) {
            runRegistryGC();
        }
    }
}

Example 24 with ServiceException

use of io.hops.hopsworks.exceptions.ServiceException in project hopsworks by logicalclocks.

the class LibraryInstaller method createNewImage.

private void createNewImage(CondaCommands cc) throws IOException, ServiceDiscoveryException {
    File baseDir = new File("/tmp/docker/" + cc.getProjectId().getName());
    baseDir.mkdirs();
    try {
        File home = new File(System.getProperty("user.home"));
        File condarc = new File(home, ".condarc");
        File pip = new File(home, ".pip");
        FileUtils.copyFileToDirectory(condarc, baseDir);
        FileUtils.copyDirectoryToDirectory(pip, baseDir);
        File dockerFile = new File(baseDir, "dockerFile_" + cc.getProjectId().getName());
        try (BufferedWriter writer = new BufferedWriter(new FileWriter(dockerFile))) {
            String baseImage = projectUtils.getFullBaseImageName();
            // image that does not contain a python environment
            if (!Strings.isNullOrEmpty(cc.getEnvironmentFile()) && !cc.getInstallJupyter()) {
                baseImage = baseImage.replace(settings.getBaseDockerImagePythonName(), settings.getBaseNonPythonDockerImage());
            }
            writer.write("# syntax=docker/dockerfile:experimental");
            writer.newLine();
            writer.write("FROM " + baseImage);
            writer.newLine();
            // copy it in the image, create the env and delete it
            if (!Strings.isNullOrEmpty(cc.getEnvironmentFile())) {
                writer.write("RUN rm -f /root/.condarc");
                writer.newLine();
                // Materialize IMPORT FILE
                String environmentFilePath = cc.getEnvironmentFile();
                String environmentFile = FilenameUtils.getName(environmentFilePath);
                writer.write("COPY .condarc .pip " + environmentFile + " /root/");
                writer.newLine();
                copyCondaArtifactToLocal(environmentFilePath, baseDir + File.separator + environmentFile);
                if (environmentFilePath.endsWith(".yml")) {
                    if (cc.getInstallJupyter()) {
                        writer.write("RUN conda env update -f /root/" + environmentFile + " -n " + settings.getCurrentCondaEnvironment());
                    } else {
                        writer.write("RUN conda env create -f /root/" + environmentFile + " -p " + anaconda_project_dir);
                    }
                } else if (environmentFilePath.endsWith("/requirements.txt")) {
                    if (cc.getInstallJupyter()) {
                        writer.write("RUN pip install -r /root/" + environmentFile);
                    } else {
                        writer.write("RUN conda create -y -p " + anaconda_project_dir + " python=" + settings.getDockerBaseImagePythonVersion() + " && pip install -r /root/" + environmentFile);
                    }
                }
                writer.write(" && " + getCleanupCommand() + " && " + anaconda_dir + "/bin/conda list -n " + settings.getCurrentCondaEnvironment());
            }
        }
        Project project = projectFacade.findById(cc.getProjectId().getId()).orElseThrow(() -> new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "projectId: " + cc.getProjectId().getId()));
        String initialDockerImage = projectUtils.getInitialDockerImageName(project);
        LOG.log(Level.FINEST, "project-initialDockerImage:" + initialDockerImage);
        ProcessDescriptor processDescriptor = new ProcessDescriptor.Builder().addCommand("/usr/bin/sudo").addCommand(prog).addCommand("create").addCommand(dockerFile.getAbsolutePath()).addCommand(projectUtils.getRegistryURL() + "/" + initialDockerImage).redirectErrorStream(true).setCurrentWorkingDirectory(baseDir).setWaitTimeout(1, TimeUnit.HOURS).build();
        ProcessResult processResult = osProcessExecutor.execute(processDescriptor);
        if (processResult.getExitCode() != 0) {
            String errorMsg = "Could not create the docker image. Exit code: " + processResult.getExitCode() + " out: " + processResult.getStdout() + "\n err: " + processResult.getStderr() + "||\n";
            throw new IOException(errorMsg);
        } else {
            project = projectFacade.findById(cc.getProjectId().getId()).orElseThrow(() -> new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "projectId: " + cc.getProjectId().getId()));
            project.setDockerImage(initialDockerImage);
            project = projectFacade.update(project);
            projectFacade.flushEm();
            setPipConflicts(project);
            environmentController.updateInstalledDependencies(project);
            exportEnvironment(project, cc.getUserId(), Utils.getProjectPath(project.getName()) + Settings.PROJECT_PYTHON_ENVIRONMENT_FILE);
        }
    } catch (ServiceException | ProjectException | PythonException e) {
        LOG.log(Level.SEVERE, "Failed to persist python deps", e);
    } finally {
        FileUtils.deleteDirectory(baseDir);
    }
}

Example 25 with ServiceException

use of io.hops.hopsworks.exceptions.ServiceException in project hopsworks by logicalclocks.

the class EnvironmentController method uploadYmlInProject.

public void uploadYmlInProject(Project project, Users user, String environmentYml, String exportPath) throws ServiceException {
    DistributedFileSystemOps udfso = null;
    String hdfsUser = hdfsUsersController.getHdfsUserName(project, user);
    try {
        udfso = dfs.getDfsOps(hdfsUser);
        Path projectYmlPath = new Path(exportPath);
        udfso.create(projectYmlPath, environmentYml);
    } catch (IOException ex) {
        throw new ServiceException(RESTCodes.ServiceErrorCode.ANACONDA_EXPORT_ERROR, Level.SEVERE, "path: " + exportPath, ex.getMessage(), ex);
    } finally {
        if (udfso != null) {
            dfs.closeDfsClient(udfso);
        }
    }
}