Examples with Dataset io.hops.hopsworks.persistence.entity.dataset.Dataset used on opensource projects

Example 16 with Dataset

use of io.hops.hopsworks.persistence.entity.dataset.Dataset in project hopsworks by logicalclocks.

the class PathValidator method buildProjectDsRelativePath.

private void buildProjectDsRelativePath(Project project, String[] pathComponents, DsPath dsPath) throws ProjectException, DatasetException {
    // Start by 1 as the first component is ""
    Project destProject = projectFacade.findByName(pathComponents[2]);
    if (project == null || destProject == null) {
        throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE);
    }
    Dataset ds = datasetController.getByProjectAndDsName(project, Utils.getProjectPath(pathComponents[2]), pathComponents[3]);
    if (ds == null) {
        throw new DatasetException(RESTCodes.DatasetErrorCode.DATASET_NOT_FOUND, Level.FINE);
    }
    dsPath.setDs(ds);
    String dsRelativePathStr = buildRelativePath(pathComponents, 4, pathComponents.length);
    if (!dsRelativePathStr.isEmpty()) {
        dsPath.setDsRelativePath(new Path(dsRelativePathStr));
    }
}

Example 17 with Dataset

use of io.hops.hopsworks.persistence.entity.dataset.Dataset in project hopsworks by logicalclocks.

the class PathValidator method buildFullPath.

private void buildFullPath(Project project, String path, DsPath dsPath) throws DatasetException {
    // Strip leading slashes.
    while (path.startsWith("/")) {
        path = path.substring(1);
    }
    String[] pathComponents = path.split(File.separator);
    String dsName = pathComponents[0];
    boolean shared = false;
    String parentProjectPath = null;
    if (pathComponents[0].contains(Settings.SHARED_FILE_SEPARATOR)) {
        // we can split the string and get the project name
        String[] shardDS = pathComponents[0].split(Settings.SHARED_FILE_SEPARATOR);
        if (shardDS.length != 2) {
            throw new DatasetException(RESTCodes.DatasetErrorCode.DATASET_NOT_FOUND, Level.FINE);
        }
        parentProjectPath = Utils.getProjectPath(shardDS[0]);
        dsName = shardDS[1];
        shared = true;
    }
    Dataset ds = datasetController.getByProjectAndDsName(project, parentProjectPath, dsName);
    if (ds == null) {
        throw new DatasetException(RESTCodes.DatasetErrorCode.DATASET_NOT_FOUND, Level.FINE);
    }
    // If the dataset is shared, make sure that the user can access it
    if (shared) {
        DatasetSharedWith datasetSharedWith = datasetSharedWithFacade.findByProjectAndDataset(project, ds);
        if (datasetSharedWith != null && !datasetSharedWith.getAccepted()) {
            throw new DatasetException(RESTCodes.DatasetErrorCode.DATASET_PENDING, Level.FINE, "datasetId: " + ds.getId());
        }
    }
    dsPath.setDs(ds);
    String dsRelativePathStr = buildRelativePath(pathComponents, 1, pathComponents.length);
    if (dsRelativePathStr.isEmpty()) {
        dsPath.setFullPath(datasetController.getDatasetPath(ds));
    } else {
        Path dsRelativePath = new Path(dsRelativePathStr);
        dsPath.setDsRelativePath(dsRelativePath);
        Path fullPath = new Path(datasetController.getDatasetPath(ds), dsRelativePath);
        dsPath.setFullPath(fullPath);
    }
}

Example 18 with Dataset

use of io.hops.hopsworks.persistence.entity.dataset.Dataset in project hopsworks by logicalclocks.

the class PathValidator method buildHiveDsRelativePath.

private void buildHiveDsRelativePath(Project project, String[] pathComponents, DsPath dsPath) throws DatasetException {
    String dsPathStr = File.separator + buildRelativePath(pathComponents, 1, 5);
    Inode dsInode = inodeController.getInodeAtPath(dsPathStr);
    if (dsInode == null) {
        throw new DatasetException(RESTCodes.DatasetErrorCode.INODE_NOT_FOUND, Level.FINE);
    }
    Dataset originalDataset = datasetFacade.findByInode(dsInode);
    if (originalDataset == null) {
        throw new DatasetException(RESTCodes.DatasetErrorCode.DATASET_NOT_FOUND, Level.FINE);
    }
    dsPath.setDs(originalDataset);
    String dsRelativePathStr = buildRelativePath(pathComponents, 5, pathComponents.length);
    if (!dsRelativePathStr.isEmpty()) {
        dsPath.setDsRelativePath(new Path(dsRelativePathStr));
    }
}

Example 19 with Dataset

use of io.hops.hopsworks.persistence.entity.dataset.Dataset in project hopsworks by logicalclocks.

the class ProjectService method getDatasetInfo.

@GET
@Path("getDatasetInfo/{inodeId}")
@Produces(MediaType.APPLICATION_JSON)
public Response getDatasetInfo(@PathParam("inodeId") Long inodeId, @Context SecurityContext sc) throws DatasetException {
    Inode inode = inodes.findById(inodeId);
    Project proj = datasetController.getOwningProject(inode);
    Dataset ds = datasetFacade.findByProjectAndInode(proj, inode);
    if (ds == null) {
        throw new DatasetException(RESTCodes.DatasetErrorCode.DATASET_NOT_FOUND, Level.FINE, "inodeId: " + inodeId);
    }
    Collection<DatasetSharedWith> projectsContainingInode = proj.getDatasetSharedWithCollection();
    List<String> sharedWith = new ArrayList<>();
    for (DatasetSharedWith d : projectsContainingInode) {
        if (!d.getProject().getId().equals(proj.getId())) {
            sharedWith.add(d.getProject().getName());
        }
    }
    DataSetDTO dataset = new DataSetDTO(ds, proj, sharedWith);
    return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(dataset).build();
}

Example 20 with Dataset

use of io.hops.hopsworks.persistence.entity.dataset.Dataset in project hopsworks by logicalclocks.

the class ProjectService method datasetInfo.

private MoreInfoDTO datasetInfo(Long inodeId) {
    Inode inode = inodes.findById(inodeId);
    if (inode == null) {
        return null;
    }
    Dataset ds = datasetFacade.findByInode(inode);
    if (ds != null && !ds.isSearchable()) {
        return null;
    }
    MoreInfoDTO info = new MoreInfoDTO(inode);
    Users user = userFacade.findByUsername(info.getUser());
    info.setUser(user.getFname() + " " + user.getLname());
    info.setSize(inodeController.getSize(inode));
    info.setPath(inodeController.getPath(inode));
    return info;
}