Examples with DatasetPath io.hops.hopsworks.common.dataset.util.DatasetPath used on opensource projects

Example 11 with DatasetPath

use of io.hops.hopsworks.common.dataset.util.DatasetPath in project hopsworks by logicalclocks.

the class ProjectMembersService method getDatasetMembers.

@GET
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@Path("/dataset/{name}")
public Response getDatasetMembers(@PathParam("name") String dsName, @QueryParam("type") DatasetType datasetType) throws ProjectException, DatasetException {
    Project project = projectController.findProjectById(this.projectId);
    String path = Utils.getProjectPath(project.getName()) + dsName;
    DatasetPath dp = datasetHelper.getDatasetPath(project, path, datasetType);
    Collection<ProjectTeam> membersCol = accessCtrl.getExtendedMembers(dp.getDataset());
    GenericEntity<Collection<ProjectTeam>> members = new GenericEntity<Collection<ProjectTeam>>(membersCol) {
    };
    return Response.ok().entity(members).build();
}

Example 12 with DatasetPath

use of io.hops.hopsworks.common.dataset.util.DatasetPath in project hopsworks by logicalclocks.

the class ProjectProvenanceResource method usage.

@GET
@Path("usage")
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER })
@JWTRequired(acceptedTokens = { Audience.API }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.PROJECT }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiOperation(value = "Artifact usage", response = ProvArtifactUsageParentDTO.class)
public Response usage(@QueryParam("artifact_id") String artifactId, @QueryParam("endpoint_id") Integer endpointId, @QueryParam("artifact_type") DatasetAccessType accessType, @BeanParam ProvUsageBeanParams params, @Context UriInfo uriInfo, @Context SecurityContext sc) throws ProvenanceException, GenericException, DatasetException, MetadataException, SchematizedTagException {
    Users user = jWTHelper.getUserPrincipal(sc);
    if (artifactId == null) {
        throw new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.FINE, "artifactId id cannot be null");
    }
    Project targetProject = project;
    if (endpointId != null) {
        targetProject = projectFacade.findById(endpointId).orElseThrow(() -> new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.FINE, "target project not found"));
    }
    Dataset targetEndpoint;
    if (accessType != null) {
        try {
            switch(accessType) {
                case FEATUREGROUPS:
                    targetEndpoint = fsCtrl.getProjectFeaturestoreDataset(targetProject);
                    break;
                case TRAININGDATASETS:
                    String tdName = project.getName() + "_" + Settings.ServiceDataset.TRAININGDATASETS.getName();
                    targetEndpoint = datasetCtrl.getByName(targetProject, tdName);
                    break;
                case MODELS:
                    targetEndpoint = datasetCtrl.getByName(targetProject, Settings.HOPS_MODELS_DATASET);
                    break;
                case EXPERIMENTS:
                    targetEndpoint = datasetCtrl.getByName(targetProject, Settings.HOPS_EXPERIMENTS_DATASET);
                    break;
                default:
                    throw new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.FINE, "access type not supports:" + accessType);
            }
        } catch (FeaturestoreException | DatasetException e) {
            throw new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_STATE, Level.FINE, "cannot access the dataset of the artifact");
        }
    } else {
        throw new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_STATE, Level.FINE, "access type not defined");
    }
    DatasetPath targetEndpointPath = datasetHelper.getTopLevelDatasetPath(project, targetEndpoint);
    ProvArtifactUsageParentDTO status = usageBuilder.buildAccessible(uriInfo, user, targetEndpointPath, artifactId, params.getUsageType());
    return Response.ok().entity(status).build();
}

Example 13 with DatasetPath

use of io.hops.hopsworks.common.dataset.util.DatasetPath in project hopsworks by logicalclocks.

the class DownloadService method downloadFromHDFS.

@GET
@javax.ws.rs.Path("with_token/{path: .+}")
@Produces(MediaType.APPLICATION_OCTET_STREAM)
@JWTNotRequired
@ApiOperation(value = "Download file.", response = StreamingOutput.class)
public Response downloadFromHDFS(@PathParam("path") String path, @QueryParam("token") String token, @QueryParam("type") DatasetType datasetType, @Context SecurityContext sc) throws DatasetException, SigningKeyNotFoundException, VerificationException, ProjectException {
    if (!settings.isDownloadAllowed()) {
        throw new DatasetException(RESTCodes.DatasetErrorCode.DOWNLOAD_NOT_ALLOWED, Level.FINEST);
    }
    Project project = this.getProject();
    DatasetPath datasetPath = datasetHelper.getDatasetPathIfFileExist(project, path, datasetType);
    String fullPath = datasetPath.getFullPath().toString();
    DecodedJWT djwt = jWTHelper.verifyOneTimeToken(token, fullPath);
    Users user = userFacade.findByUsername(djwt.getSubject());
    Pair<Path, StreamingOutput> pathStreamPair = downloadFromHDFS(project, datasetPath, user);
    Response.ResponseBuilder response = Response.ok(pathStreamPair.getValue1());
    response.header("Content-disposition", "attachment; filename=\"" + pathStreamPair.getValue0().getName() + "\"");
    return response.build();
}

Example 14 with DatasetPath

use of io.hops.hopsworks.common.dataset.util.DatasetPath in project hopsworks by logicalclocks.

the class DownloadService method getDownloadToken.

@GET
@javax.ws.rs.Path("token/{path: .+}")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Get a one time download token.", response = RESTApiJsonResponse.class)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER })
@JWTRequired(acceptedTokens = { Audience.API }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response getDownloadToken(@PathParam("path") String path, @QueryParam("type") DatasetType datasetType, @Context SecurityContext sc) throws DatasetException, ProjectException {
    if (!settings.isDownloadAllowed()) {
        throw new DatasetException(RESTCodes.DatasetErrorCode.DOWNLOAD_NOT_ALLOWED, Level.FINEST);
    }
    Users user = jWTHelper.getUserPrincipal(sc);
    DatasetPath datasetPath = datasetHelper.getDatasetPathIfFileExist(this.getProject(), path, datasetType);
    Project owningProject = datasetController.getOwningProject(datasetPath.getDataset());
    RESTApiJsonResponse response = new RESTApiJsonResponse();
    String username = hdfsUsersController.getHdfsUserName(this.getProject(), user);
    // For example, DS1 of project1 is shared with project2. User must be a member of project1 to download files
    if (owningProject.equals(this.getProject()) && datasetController.isDownloadAllowed(this.getProject(), user, datasetPath.getFullPath().toString())) {
        datasetController.checkFileExists(datasetPath.getFullPath(), username);
        String token = jWTHelper.createOneTimeToken(user, datasetPath.getFullPath().toString(), null);
        if (token != null && !token.isEmpty()) {
            response.setData(token);
            return Response.status(Response.Status.OK).entity(response).build();
        }
    }
    response.setErrorMsg(ResponseMessages.DOWNLOAD_PERMISSION_ERROR);
    return Response.status(Response.Status.FORBIDDEN).entity(response).build();
}

Example 15 with DatasetPath

use of io.hops.hopsworks.common.dataset.util.DatasetPath in project hopsworks by logicalclocks.

the class DownloadService method downloadFromHDFS.

/**
 * @param project
 * @param datasetPath
 * @param user
 * @return
 */
private Pair<Path, StreamingOutput> downloadFromHDFS(Project project, DatasetPath datasetPath, Users user) throws DatasetException {
    String fullPath = datasetPath.getFullPath().toString();
    String projectUsername = hdfsUsersController.getHdfsUserName(project, user);
    Dataset ds = datasetPath.getDataset();
    if (ds.isShared(project) && ds.getFilePermissions().equals(DatasetPermissions.OWNER_ONLY) && !ds.isPublicDs()) {
        throw new DatasetException(RESTCodes.DatasetErrorCode.DOWNLOAD_ERROR, Level.FINE);
    }
    FSDataInputStream stream;
    DistributedFileSystemOps udfso;
    try {
        if (projectUsername != null) {
            udfso = dfs.getDfsOps(projectUsername);
            Path p = new Path(fullPath);
            stream = udfso.open(p);
            return new Pair(p, buildOutputStream(stream, udfso));
        } else {
            throw new DatasetException(RESTCodes.DatasetErrorCode.DOWNLOAD_ERROR, Level.WARNING);
        }
    } catch (IOException ex) {
        throw new DatasetException(RESTCodes.DatasetErrorCode.DOWNLOAD_ERROR, Level.SEVERE, "path: " + fullPath, ex.getMessage(), ex);
    }
}