Examples with ModelRegistryDTO io.hops.hopsworks.api.modelregistry.dto.ModelRegistryDTO used on opensource projects

Example 1 with ModelRegistryDTO

use of io.hops.hopsworks.api.modelregistry.dto.ModelRegistryDTO in project hopsworks by logicalclocks.

the class ModelsBuilder method buildFilter.

private Pair<ProvStateParamBuilder, ModelRegistryDTO> buildFilter(Project project, Project modelRegistryProject, Set<? extends AbstractFacade.FilterBy> filters) throws GenericException, ProvenanceException, DatasetException {
    ProvStateParamBuilder provFilesParamBuilder = new ProvStateParamBuilder();
    if (filters != null) {
        Users filterUser = null;
        Project filterUserProject = project;
        for (AbstractFacade.FilterBy filterBy : filters) {
            if (filterBy.getParam().compareToIgnoreCase(Filters.NAME_EQ.name()) == 0) {
                provFilesParamBuilder.filterByXAttr(MODEL_SUMMARY_XATTR_NAME + ".name", filterBy.getValue());
            } else if (filterBy.getParam().compareToIgnoreCase(Filters.NAME_LIKE.name()) == 0) {
                provFilesParamBuilder.filterLikeXAttr(MODEL_SUMMARY_XATTR_NAME + ".name", filterBy.getValue());
            } else if (filterBy.getParam().compareToIgnoreCase(Filters.VERSION.name()) == 0) {
                provFilesParamBuilder.filterByXAttr(MODEL_SUMMARY_XATTR_NAME + ".version", filterBy.getValue());
            } else if (filterBy.getParam().compareToIgnoreCase(Filters.ID_EQ.name()) == 0) {
                provFilesParamBuilder.filterByXAttr(MODEL_SUMMARY_XATTR_NAME + ".id", filterBy.getValue());
            } else if (filterBy.getParam().compareToIgnoreCase(Filters.USER.name()) == 0) {
                try {
                    filterUser = userFacade.find(Integer.parseInt(filterBy.getValue()));
                } catch (NumberFormatException e) {
                    throw new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.INFO, "expected int user id, found: " + filterBy.getValue());
                }
            } else if (filterBy.getParam().compareToIgnoreCase(Filters.USER_PROJECT.name()) == 0) {
                try {
                    filterUserProject = projectFacade.find(Integer.parseInt(filterBy.getValue()));
                } catch (NumberFormatException e) {
                    throw new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.INFO, "expected int user project id, found: " + filterBy.getValue());
                }
            } else {
                throw new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.INFO, "Filter by - found: " + filterBy.getParam() + " expected:" + EnumSet.allOf(Filters.class));
            }
        }
        if (filterUser != null) {
            ProjectTeam member = projectTeamFacade.findByPrimaryKey(filterUserProject, filterUser);
            if (member == null) {
                throw new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.INFO, "Selected user: " + filterUser.getUid() + " is not part of project:" + filterUserProject.getId());
            }
            String hdfsUserStr = hdfsUsersController.getHdfsUserName(filterUserProject, filterUser);
            HdfsUsers hdfsUsers = hdfsUsersFacade.findByName(hdfsUserStr);
            provFilesParamBuilder.filterByField(ProvStateParser.FieldsP.USER_ID, hdfsUsers.getId().toString());
        }
    }
    ModelRegistryDTO modelRegistryDTO = modelsController.getModelRegistry(modelRegistryProject);
    provFilesParamBuilder.filterByField(ProvStateParser.FieldsP.PROJECT_I_ID, modelRegistryDTO.getParentProject().getInode().getId()).filterByField(ProvStateParser.FieldsP.DATASET_I_ID, modelRegistryDTO.getDatasetInodeId());
    return Pair.with(provFilesParamBuilder, modelRegistryDTO);
}

Example 2 with ModelRegistryDTO

use of io.hops.hopsworks.api.modelregistry.dto.ModelRegistryDTO in project hopsworks by logicalclocks.

the class ModelRegistryBuilder method build.

// Build collection
public ModelRegistryDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, Users user, Project project) throws GenericException, ModelRegistryException, SchematizedTagException, MetadataException {
    ModelRegistryDTO dto = new ModelRegistryDTO();
    uri(dto, uriInfo, project);
    expand(dto, resourceRequest);
    Collection<Dataset> dsInProject = project.getDatasetCollection();
    // Add all datasets shared with the project
    dsInProject.addAll(project.getDatasetSharedWithCollection().stream().filter(DatasetSharedWith::getAccepted).map(DatasetSharedWith::getDataset).collect(Collectors.toList()));
    Collection<Dataset> modelsDatasets = dsInProject.stream().filter(ds -> ds.getName().equals(Settings.HOPS_MODELS_DATASET)).collect(Collectors.toList());
    dto.setCount((long) modelsDatasets.size());
    for (Dataset ds : modelsDatasets) {
        ModelRegistryDTO modelRegistryDTO = build(uriInfo, resourceRequest, user, project, ds.getProject());
        if (modelRegistryDTO != null) {
            dto.addItem(modelRegistryDTO);
        }
    }
    return dto;
}

Example 3 with ModelRegistryDTO

use of io.hops.hopsworks.api.modelregistry.dto.ModelRegistryDTO in project hopsworks by logicalclocks.

the class ModelsBuilder method build.

// Build collection
public ModelDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, Users user, Project userProject, Project modelRegistryProject) throws ModelRegistryException, GenericException, SchematizedTagException, MetadataException {
    ModelDTO dto = new ModelDTO();
    uri(dto, uriInfo, userProject, modelRegistryProject);
    expand(dto, resourceRequest);
    dto.setCount(0l);
    if (dto.isExpand()) {
        validatePagination(resourceRequest);
        ProvStateDTO fileState;
        try {
            Pair<ProvStateParamBuilder, ModelRegistryDTO> provFilesParamBuilder = buildModelProvenanceParams(userProject, modelRegistryProject, resourceRequest);
            if (provFilesParamBuilder.getValue1() == null) {
                // no endpoint - no results
                return dto;
            }
            fileState = provenanceController.provFileStateList(provFilesParamBuilder.getValue1().getParentProject(), provFilesParamBuilder.getValue0());
            List<ProvStateDTO> models = new LinkedList<>(fileState.getItems());
            dto.setCount(fileState.getCount());
            String modelsDatasetPath = modelUtils.getModelsDatasetPath(userProject, modelRegistryProject);
            for (ProvStateDTO fileProvStateHit : models) {
                ModelDTO modelDTO = build(uriInfo, resourceRequest, user, userProject, modelRegistryProject, fileProvStateHit, modelsDatasetPath);
                if (modelDTO != null) {
                    dto.addItem(modelDTO);
                }
            }
        } catch (ProvenanceException e) {
            if (ProvHelper.missingMappingForField(e)) {
                LOGGER.log(Level.WARNING, "Could not find elastic mapping for experiments query", e);
                return dto;
            } else {
                throw new ModelRegistryException(RESTCodes.ModelRegistryErrorCode.MODEL_LIST_FAILED, Level.FINE, "Unable to list models for project " + modelRegistryProject.getName(), e.getMessage(), e);
            }
        } catch (DatasetException e) {
            throw new ModelRegistryException(RESTCodes.ModelRegistryErrorCode.MODEL_LIST_FAILED, Level.FINE, "Unable to list models for project " + modelRegistryProject.getName(), e.getMessage(), e);
        }
    }
    return dto;
}

Example 4 with ModelRegistryDTO

use of io.hops.hopsworks.api.modelregistry.dto.ModelRegistryDTO in project hopsworks by logicalclocks.

the class ModelRegistryResource method getAll.

@ApiOperation(value = "Get a list of all model registries accessible for this project", response = ModelRegistryDTO.class)
@GET
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.MODELREGISTRY }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response getAll(@BeanParam ModelRegistryBeanParam modelRegistryBeanParam, @BeanParam Pagination pagination, @Context UriInfo uriInfo, @Context SecurityContext sc) throws GenericException, ModelRegistryException, SchematizedTagException, MetadataException {
    Users user = jwtHelper.getUserPrincipal(sc);
    ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.MODELREGISTRIES);
    resourceRequest.setExpansions(modelRegistryBeanParam.getExpansions().getResources());
    resourceRequest.setOffset(pagination.getOffset());
    resourceRequest.setLimit(pagination.getLimit());
    ModelRegistryDTO dto = modelRegistryBuilder.build(uriInfo, resourceRequest, user, project);
    return Response.ok().entity(dto).build();
}

Example 5 with ModelRegistryDTO

use of io.hops.hopsworks.api.modelregistry.dto.ModelRegistryDTO in project hopsworks by logicalclocks.

the class ModelRegistryResource method get.

@ApiOperation(value = "Get a model registry", response = ModelDTO.class)
@GET
@Path("{modelRegistryId}")
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.MODELREGISTRY }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response get(@PathParam("modelRegistryId") Integer modelRegistryId, @BeanParam ModelRegistryBeanParam modelRegistryBeanParam, @BeanParam Pagination pagination, @Context UriInfo uriInfo, @Context SecurityContext sc) throws GenericException, ModelRegistryException, SchematizedTagException, MetadataException {
    Users user = jwtHelper.getUserPrincipal(sc);
    ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.MODELREGISTRIES);
    resourceRequest.setExpansions(modelRegistryBeanParam.getExpansions().getResources());
    Project modelRegistryProject = modelsController.verifyModelRegistryAccess(project, modelRegistryId).getParentProject();
    ModelRegistryDTO dto = modelRegistryBuilder.build(uriInfo, resourceRequest, user, project, modelRegistryProject);
    return Response.ok().entity(dto).build();
}