Examples with DatasetException io.hops.hopsworks.exceptions.DatasetException used on opensource projects

Example 26 with DatasetException

use of io.hops.hopsworks.exceptions.DatasetException in project hopsworks by logicalclocks.

the class ProjectController method createProject.

/**
 * Creates a new project(project), the related DIR, the different services in
 * the project, and the master of the
 * project.
 * <p>
 * This needs to be an atomic operation (all or nothing) REQUIRES_NEW will
 * make sure a new transaction is created even
 * if this method is called from within a transaction.
 *
 * @param projectDTO
 * @param owner
 * @param sessionId
 * @return
 */
public Project createProject(ProjectDTO projectDTO, Users owner, String sessionId) throws DatasetException, GenericException, KafkaException, ProjectException, UserException, HopsSecurityException, ServiceException, FeaturestoreException, ElasticException, SchemaException, IOException {
    Long startTime = System.currentTimeMillis();
    // check that the project name is ok
    String projectName = projectDTO.getProjectName();
    FolderNameValidator.isValidProjectName(projectUtils, projectName);
    List<ProjectServiceEnum> projectServices = new ArrayList<>();
    if (projectDTO.getServices() != null) {
        for (String s : projectDTO.getServices()) {
            ProjectServiceEnum se = ProjectServiceEnum.valueOf(s.toUpperCase());
            projectServices.add(se);
        }
    }
    LOGGER.log(Level.FINE, () -> "PROJECT CREATION TIME. Step 1: " + (System.currentTimeMillis() - startTime));
    DistributedFileSystemOps dfso = null;
    Project project = null;
    try {
        dfso = dfs.getDfsOps();
        /*
       * create a project in the database
       * if the creation go through it means that there is no other project with
       * the same name.
       * this project creation act like a lock, no other project can be created
       * with the same name
       * until this project is removed from the database
       */
        try {
            project = createProject(projectName, owner, projectDTO.getDescription(), dfso);
        } catch (EJBException ex) {
            LOGGER.log(Level.WARNING, null, ex);
            Path dummy = new Path("/tmp/" + projectName);
            try {
                dfso.rm(dummy, true);
            } catch (IOException e) {
                LOGGER.log(Level.SEVERE, null, e);
            }
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_EXISTS, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 2 (hdfs): {0}", System.currentTimeMillis() - startTime);
        verifyProject(project, dfso, sessionId);
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 3 (verify): {0}", System.currentTimeMillis() - startTime);
        // Run the handlers.
        try {
            ProjectHandler.runProjectPreCreateHandlers(projectHandlers, project);
        } catch (ProjectException ex) {
            cleanup(project, sessionId, null, true, owner);
            throw ex;
        }
        List<Future<?>> projectCreationFutures = new ArrayList<>();
        // This is an async call
        try {
            projectCreationFutures.add(certificatesController.generateCertificates(project, owner));
        } catch (Exception ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new HopsSecurityException(RESTCodes.SecurityErrorCode.CERT_CREATION_ERROR, Level.SEVERE, "project: " + project.getName() + "owner: " + owner.getUsername(), ex.getMessage(), ex);
        }
        String username = hdfsUsersController.getHdfsUserName(project, owner);
        if (username == null || username.isEmpty()) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new UserException(RESTCodes.UserErrorCode.USER_WAS_NOT_FOUND, Level.SEVERE, "project: " + project.getName() + "owner: " + owner.getUsername());
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 4 (certs): {0}", System.currentTimeMillis() - startTime);
        // all the verifications have passed, we can now create the project
        // create the project folder
        ProvTypeDTO provType = settings.getProvType().dto;
        try {
            mkProjectDIR(projectName, dfso);
            fsProvController.updateProjectProvType(project, provType, dfso);
        } catch (IOException | EJBException | ProvenanceException ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_FOLDER_NOT_CREATED, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 5 (folders): {0}", System.currentTimeMillis() - startTime);
        // update the project with the project folder inode
        try {
            setProjectInode(project, dfso);
        } catch (IOException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_INODE_CREATION_ERROR, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 6 (inodes): {0}", System.currentTimeMillis() - startTime);
        // set payment and quotas
        try {
            setProjectOwnerAndQuotas(project, dfso, owner);
        } catch (IOException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new ProjectException(RESTCodes.ProjectErrorCode.QUOTA_ERROR, Level.SEVERE, "project: " + project.getName(), ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 7 (quotas): {0}", System.currentTimeMillis() - startTime);
        try {
            hdfsUsersController.addProjectFolderOwner(project, dfso);
            createProjectLogResources(owner, project, dfso);
        } catch (IOException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_SET_PERMISSIONS_ERROR, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 8 (logs): {0}", System.currentTimeMillis() - startTime);
        // inconsistencies
        try {
            elasticController.deleteProjectIndices(project);
            elasticController.deleteProjectSavedObjects(projectName);
            LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 9 (elastic cleanup): {0}", System.currentTimeMillis() - startTime);
        } catch (ElasticException ex) {
            LOGGER.log(Level.FINE, "Error while cleaning old project indices", ex);
        }
        logProject(project, OperationType.Add);
        // enable services
        for (ProjectServiceEnum service : projectServices) {
            try {
                projectCreationFutures.addAll(addService(project, service, owner, dfso, provType));
            } catch (RESTException | IOException ex) {
                cleanup(project, sessionId, projectCreationFutures);
                throw ex;
            }
        }
        try {
            for (Future f : projectCreationFutures) {
                if (f != null) {
                    f.get();
                }
            }
        } catch (InterruptedException | ExecutionException ex) {
            LOGGER.log(Level.SEVERE, "Error while waiting for the certificate generation thread to finish. Will try to " + "cleanup...", ex);
            cleanup(project, sessionId, projectCreationFutures);
            throw new HopsSecurityException(RESTCodes.SecurityErrorCode.CERT_CREATION_ERROR, Level.SEVERE);
        }
        // Run the handlers.
        try {
            ProjectHandler.runProjectPostCreateHandlers(projectHandlers, project);
        } catch (ProjectException ex) {
            cleanup(project, sessionId, projectCreationFutures);
            throw ex;
        }
        try {
            project = environmentController.createEnv(project, owner);
        } catch (PythonException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_ANACONDA_ENABLE_ERROR, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 10 (env): {0}", System.currentTimeMillis() - startTime);
        return project;
    } finally {
        if (dfso != null) {
            dfso.close();
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 11 (close): {0}", System.currentTimeMillis() - startTime);
    }
}

Example 27 with DatasetException

use of io.hops.hopsworks.exceptions.DatasetException in project hopsworks by logicalclocks.

the class X509Resource method getx509.

@GET
@TransactionAttribute(TransactionAttributeType.NEVER)
@Produces(MediaType.APPLICATION_JSON)
@JWTRequired(acceptedTokens = { Audience.SERVICES, Audience.API }, allowedUserRoles = { "AGENT", "HOPS_ADMIN" })
@ApiOperation(value = "Get keystore, truststore and password of a project user", response = AccessCredentialsDTO.class)
public Response getx509(@QueryParam("username") String projectUsername, @Context SecurityContext sc) throws ProjectException, UserException, HopsSecurityException {
    try {
        String projectName = hdfsUsersController.getProjectName(projectUsername);
        String username = hdfsUsersController.getUserName(projectUsername);
        Project project = projectController.findProjectByName(projectName);
        Users user = userFacade.findByUsername(username);
        if (user == null) {
            throw new UserException(RESTCodes.UserErrorCode.USER_DOES_NOT_EXIST, Level.FINE);
        }
        try {
            AccessCredentialsDTO credentialsDTO = projectController.credentials(project.getId(), user);
            return Response.ok(credentialsDTO).build();
        } catch (DatasetException ex) {
            throw new HopsSecurityException(RESTCodes.SecurityErrorCode.CERTIFICATE_NOT_FOUND, Level.FINE);
        }
    } catch (ArrayIndexOutOfBoundsException ex) {
        throw new UserException(RESTCodes.UserErrorCode.USER_WAS_NOT_FOUND, Level.FINE, "Invalid project user format for username: " + projectUsername);
    }
}

Example 28 with DatasetException

use of io.hops.hopsworks.exceptions.DatasetException in project hopsworks by logicalclocks.

the class DownloadService method downloadFromHDFS.

@GET
@javax.ws.rs.Path("with_token/{path: .+}")
@Produces(MediaType.APPLICATION_OCTET_STREAM)
@JWTNotRequired
@ApiOperation(value = "Download file.", response = StreamingOutput.class)
public Response downloadFromHDFS(@PathParam("path") String path, @QueryParam("token") String token, @QueryParam("type") DatasetType datasetType, @Context SecurityContext sc) throws DatasetException, SigningKeyNotFoundException, VerificationException, ProjectException {
    if (!settings.isDownloadAllowed()) {
        throw new DatasetException(RESTCodes.DatasetErrorCode.DOWNLOAD_NOT_ALLOWED, Level.FINEST);
    }
    Project project = this.getProject();
    DatasetPath datasetPath = datasetHelper.getDatasetPathIfFileExist(project, path, datasetType);
    String fullPath = datasetPath.getFullPath().toString();
    DecodedJWT djwt = jWTHelper.verifyOneTimeToken(token, fullPath);
    Users user = userFacade.findByUsername(djwt.getSubject());
    Pair<Path, StreamingOutput> pathStreamPair = downloadFromHDFS(project, datasetPath, user);
    Response.ResponseBuilder response = Response.ok(pathStreamPair.getValue1());
    response.header("Content-disposition", "attachment; filename=\"" + pathStreamPair.getValue0().getName() + "\"");
    return response.build();
}

Example 29 with DatasetException

use of io.hops.hopsworks.exceptions.DatasetException in project hopsworks by logicalclocks.

the class DownloadService method getDownloadToken.

@GET
@javax.ws.rs.Path("token/{path: .+}")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Get a one time download token.", response = RESTApiJsonResponse.class)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER })
@JWTRequired(acceptedTokens = { Audience.API }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response getDownloadToken(@PathParam("path") String path, @QueryParam("type") DatasetType datasetType, @Context SecurityContext sc) throws DatasetException, ProjectException {
    if (!settings.isDownloadAllowed()) {
        throw new DatasetException(RESTCodes.DatasetErrorCode.DOWNLOAD_NOT_ALLOWED, Level.FINEST);
    }
    Users user = jWTHelper.getUserPrincipal(sc);
    DatasetPath datasetPath = datasetHelper.getDatasetPathIfFileExist(this.getProject(), path, datasetType);
    Project owningProject = datasetController.getOwningProject(datasetPath.getDataset());
    RESTApiJsonResponse response = new RESTApiJsonResponse();
    String username = hdfsUsersController.getHdfsUserName(this.getProject(), user);
    // For example, DS1 of project1 is shared with project2. User must be a member of project1 to download files
    if (owningProject.equals(this.getProject()) && datasetController.isDownloadAllowed(this.getProject(), user, datasetPath.getFullPath().toString())) {
        datasetController.checkFileExists(datasetPath.getFullPath(), username);
        String token = jWTHelper.createOneTimeToken(user, datasetPath.getFullPath().toString(), null);
        if (token != null && !token.isEmpty()) {
            response.setData(token);
            return Response.status(Response.Status.OK).entity(response).build();
        }
    }
    response.setErrorMsg(ResponseMessages.DOWNLOAD_PERMISSION_ERROR);
    return Response.status(Response.Status.FORBIDDEN).entity(response).build();
}

Example 30 with DatasetException

use of io.hops.hopsworks.exceptions.DatasetException in project hopsworks by logicalclocks.

the class DownloadService method downloadFromHDFS.

/**
 * @param project
 * @param datasetPath
 * @param user
 * @return
 */
private Pair<Path, StreamingOutput> downloadFromHDFS(Project project, DatasetPath datasetPath, Users user) throws DatasetException {
    String fullPath = datasetPath.getFullPath().toString();
    String projectUsername = hdfsUsersController.getHdfsUserName(project, user);
    Dataset ds = datasetPath.getDataset();
    if (ds.isShared(project) && ds.getFilePermissions().equals(DatasetPermissions.OWNER_ONLY) && !ds.isPublicDs()) {
        throw new DatasetException(RESTCodes.DatasetErrorCode.DOWNLOAD_ERROR, Level.FINE);
    }
    FSDataInputStream stream;
    DistributedFileSystemOps udfso;
    try {
        if (projectUsername != null) {
            udfso = dfs.getDfsOps(projectUsername);
            Path p = new Path(fullPath);
            stream = udfso.open(p);
            return new Pair(p, buildOutputStream(stream, udfso));
        } else {
            throw new DatasetException(RESTCodes.DatasetErrorCode.DOWNLOAD_ERROR, Level.WARNING);
        }
    } catch (IOException ex) {
        throw new DatasetException(RESTCodes.DatasetErrorCode.DOWNLOAD_ERROR, Level.SEVERE, "path: " + fullPath, ex.getMessage(), ex);
    }
}