Examples with Dataset io.hops.hopsworks.persistence.entity.dataset.Dataset used on opensource projects

Example 1 with Dataset

use of io.hops.hopsworks.persistence.entity.dataset.Dataset in project hopsworks by logicalclocks.

the class DatasetAccessController method accessorProjects.

private void accessorProjects(Users user, DatasetDetails datasetDetails, ProjectsCollector collector, ShortLivedCache cache) {
    // <project, userProjectRole>
    Pair<Project, String> projectAux = getProjectWithCache(user, datasetDetails.getParentProjectId(), cache);
    // check if parent project and parent dataset still exist or is this a stale item
    if (projectAux == null) {
        LOGGER.log(Level.FINE, "parent project of item - not found - probably stale item:{0}", datasetDetails);
        return;
    }
    Dataset dataset = getDatasetWithCache(projectAux.getValue0(), datasetDetails.getParentDatasetIId(), cache);
    if (dataset == null) {
        LOGGER.log(Level.FINE, "parent dataset of item - not found - probably stale item:{0}", datasetDetails);
        return;
    }
    // check parent project for access
    if (projectAux.getValue1() != null) {
        collector.addAccessProject(projectAux.getValue0());
    }
    // check shared datasets for access
    checkSharedDatasetsAccess(user, dataset, collector, cache);
}

Example 2 with Dataset

use of io.hops.hopsworks.persistence.entity.dataset.Dataset in project hopsworks by logicalclocks.

the class DelaProjectService method publish.

@POST
@Path("/uploads")
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER })
@JWTRequired(acceptedTokens = { Audience.API }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response publish(@Context SecurityContext sc, InodeIdDTO inodeId) throws DelaException {
    Inode inode = getInode(inodeId.getId());
    Dataset dataset = getDatasetByInode(inode);
    Users user = jWTHelper.getUserPrincipal(sc);
    delaWorkerCtrl.shareDatasetWithHops(project, dataset, user);
    RESTApiJsonResponse json = new RESTApiJsonResponse();
    json.setSuccessMessage("Dataset transfer is started - published");
    return successResponse(json);
}

Example 3 with Dataset

use of io.hops.hopsworks.persistence.entity.dataset.Dataset in project hopsworks by logicalclocks.

the class DelaProjectService method downloadDatasetHdfs.

@POST
@Path("/downloads/{publicDSId}/hdfs")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER })
@JWTRequired(acceptedTokens = { Audience.API }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response downloadDatasetHdfs(@Context SecurityContext sc, @PathParam("publicDSId") String publicDSId, HopsworksTransferDTO.Download downloadDTO) throws DelaException {
    Users user = jWTHelper.getUserPrincipal(sc);
    Dataset dataset = getDatasetByPublicId(downloadDTO.getPublicDSId());
    delaWorkerCtrl.advanceDownload(project, dataset, user, downloadDTO, null, null);
    return successResponse(new SuccessJSON(""));
}

Example 4 with Dataset

use of io.hops.hopsworks.persistence.entity.dataset.Dataset in project hopsworks by logicalclocks.

the class RemoteDelaService method readme.

@GET
@Path("/datasets/{publicDSId}/readme")
@Produces(MediaType.APPLICATION_JSON)
public Response readme(@PathParam("publicDSId") String publicDSId, @Context SecurityContext sc) throws DelaException {
    LOGGER.log(Settings.DELA_DEBUG, "remote:dela:readme {0}", publicDSId);
    Optional<Dataset> dataset = datasetFacade.findByPublicDsId(publicDSId);
    if (!dataset.isPresent() || !dataset.get().isPublicDs()) {
        throw new DelaException(RESTCodes.DelaErrorCode.DATASET_DOES_NOT_EXIST, Level.FINE, DelaException.Source.REMOTE_DELA);
    }
    FilePreviewDTO result = hdfsDelaCtrl.getPublicReadme(dataset.get());
    LOGGER.log(Settings.DELA_DEBUG, "remote:dela:readme - done {0}", publicDSId);
    return success(result);
}

Example 5 with Dataset

use of io.hops.hopsworks.persistence.entity.dataset.Dataset in project hopsworks by logicalclocks.

the class ElasticHitsBuilder method buildElasticInodes.

public void buildElasticInodes(SearchHit hit, ElasticInodeDTO elasticInodeDTO) {
    ElasticInodeDTO item = new ElasticInodeDTO();
    item.setMap(hit.getSourceAsMap());
    item.setScore(hit.getScore());
    item.setInodeId(Long.parseLong(hit.getId()));
    item.setParentDatasetIId(getLongValue(hit, "dataset_id"));
    item.setParentProjectId(getIntValue(hit, "project_id"));
    item.setName(getStringValue(hit, "name"));
    item.setCreator(getStringValue(hit, "user"));
    item.setDescription(getStringValue(hit, "description"));
    item.setSize(getLongValue(hit, "size"));
    item.setHighlights(hit.getHighlightFields());
    if (elasticInodeDTO.getItems() == null) {
        elasticInodeDTO.setItems(new ArrayList<>());
    }
    if (item.getParentDatasetIId() != null) {
        Dataset dataset = datasetController.getDatasetByInodeId(item.getParentDatasetIId());
        if (dataset != null) {
            item.setParentDatasetId(dataset.getId());
            item.setParentDatasetName(dataset.getName());
            item.setModificationTime(new Date(dataset.getInode().getModificationTime().longValue()));
        }
    }
    if (item.getInodeId() != null) {
        Inode inode = inodeFacade.findById(item.getInodeId());
        if (inode != null) {
            item.setPath(inodeController.getPath(inode));
        }
    }
    item.setCreator(setUserName(item.getCreator()));
    elasticInodeDTO.getItems().add(item);
}