Search in sources :

Example 1 with ProvStateParamBuilder

use of io.hops.hopsworks.common.provenance.state.ProvStateParamBuilder in project hopsworks by logicalclocks.

the class ModelsBuilder method buildFilter.

private Pair<ProvStateParamBuilder, ModelRegistryDTO> buildFilter(Project project, Project modelRegistryProject, Set<? extends AbstractFacade.FilterBy> filters) throws GenericException, ProvenanceException, DatasetException {
    ProvStateParamBuilder provFilesParamBuilder = new ProvStateParamBuilder();
    if (filters != null) {
        Users filterUser = null;
        Project filterUserProject = project;
        for (AbstractFacade.FilterBy filterBy : filters) {
            if (filterBy.getParam().compareToIgnoreCase(Filters.NAME_EQ.name()) == 0) {
                provFilesParamBuilder.filterByXAttr(MODEL_SUMMARY_XATTR_NAME + ".name", filterBy.getValue());
            } else if (filterBy.getParam().compareToIgnoreCase(Filters.NAME_LIKE.name()) == 0) {
                provFilesParamBuilder.filterLikeXAttr(MODEL_SUMMARY_XATTR_NAME + ".name", filterBy.getValue());
            } else if (filterBy.getParam().compareToIgnoreCase(Filters.VERSION.name()) == 0) {
                provFilesParamBuilder.filterByXAttr(MODEL_SUMMARY_XATTR_NAME + ".version", filterBy.getValue());
            } else if (filterBy.getParam().compareToIgnoreCase(Filters.ID_EQ.name()) == 0) {
                provFilesParamBuilder.filterByXAttr(MODEL_SUMMARY_XATTR_NAME + ".id", filterBy.getValue());
            } else if (filterBy.getParam().compareToIgnoreCase(Filters.USER.name()) == 0) {
                try {
                    filterUser = userFacade.find(Integer.parseInt(filterBy.getValue()));
                } catch (NumberFormatException e) {
                    throw new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.INFO, "expected int user id, found: " + filterBy.getValue());
                }
            } else if (filterBy.getParam().compareToIgnoreCase(Filters.USER_PROJECT.name()) == 0) {
                try {
                    filterUserProject = projectFacade.find(Integer.parseInt(filterBy.getValue()));
                } catch (NumberFormatException e) {
                    throw new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.INFO, "expected int user project id, found: " + filterBy.getValue());
                }
            } else {
                throw new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.INFO, "Filter by - found: " + filterBy.getParam() + " expected:" + EnumSet.allOf(Filters.class));
            }
        }
        if (filterUser != null) {
            ProjectTeam member = projectTeamFacade.findByPrimaryKey(filterUserProject, filterUser);
            if (member == null) {
                throw new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.INFO, "Selected user: " + filterUser.getUid() + " is not part of project:" + filterUserProject.getId());
            }
            String hdfsUserStr = hdfsUsersController.getHdfsUserName(filterUserProject, filterUser);
            HdfsUsers hdfsUsers = hdfsUsersFacade.findByName(hdfsUserStr);
            provFilesParamBuilder.filterByField(ProvStateParser.FieldsP.USER_ID, hdfsUsers.getId().toString());
        }
    }
    ModelRegistryDTO modelRegistryDTO = modelsController.getModelRegistry(modelRegistryProject);
    provFilesParamBuilder.filterByField(ProvStateParser.FieldsP.PROJECT_I_ID, modelRegistryDTO.getParentProject().getInode().getId()).filterByField(ProvStateParser.FieldsP.DATASET_I_ID, modelRegistryDTO.getDatasetInodeId());
    return Pair.with(provFilesParamBuilder, modelRegistryDTO);
}
Also used : ProvStateParamBuilder(io.hops.hopsworks.common.provenance.state.ProvStateParamBuilder) Project(io.hops.hopsworks.persistence.entity.project.Project) ProjectTeam(io.hops.hopsworks.persistence.entity.project.team.ProjectTeam) AbstractFacade(io.hops.hopsworks.common.dao.AbstractFacade) HdfsUsers(io.hops.hopsworks.persistence.entity.hdfs.user.HdfsUsers) Users(io.hops.hopsworks.persistence.entity.user.Users) ModelRegistryDTO(io.hops.hopsworks.api.modelregistry.dto.ModelRegistryDTO) GenericException(io.hops.hopsworks.exceptions.GenericException) HdfsUsers(io.hops.hopsworks.persistence.entity.hdfs.user.HdfsUsers)

Example 2 with ProvStateParamBuilder

use of io.hops.hopsworks.common.provenance.state.ProvStateParamBuilder in project hopsworks by logicalclocks.

the class ModelsController method getModel.

public ProvStateDTO getModel(Project project, String mlId) throws ProvenanceException {
    ProvStateParamBuilder provFilesParamBuilder = new ProvStateParamBuilder().filterByField(ProvStateParser.FieldsP.PROJECT_I_ID, project.getInode().getId()).filterByField(ProvStateParser.FieldsP.ML_TYPE, Provenance.MLType.MODEL.name()).filterByField(ProvStateParser.FieldsP.ML_ID, mlId).paginate(0, 1);
    ProvStateDTO fileState = provenanceController.provFileStateList(project, provFilesParamBuilder);
    if (fileState != null) {
        List<ProvStateDTO> experiments = fileState.getItems();
        if (experiments != null && !experiments.isEmpty()) {
            return experiments.iterator().next();
        }
    }
    return null;
}
Also used : ProvStateParamBuilder(io.hops.hopsworks.common.provenance.state.ProvStateParamBuilder) ProvStateDTO(io.hops.hopsworks.common.provenance.state.dto.ProvStateDTO)

Example 3 with ProvStateParamBuilder

use of io.hops.hopsworks.common.provenance.state.ProvStateParamBuilder in project hopsworks by logicalclocks.

the class ExperimentsBuilder method buildFilter.

private Pair<ProvStateParamBuilder, Map<Long, ExperimentsEndpointDTO>> buildFilter(Project project, Set<? extends AbstractFacade.FilterBy> filters) throws ProvenanceException, GenericException, DatasetException {
    ProvStateParamBuilder provFilesParamBuilder = new ProvStateParamBuilder();
    Map<Long, ExperimentsEndpointDTO> selectedEndpoints = new HashMap<>();
    if (filters != null) {
        Users filterUser = null;
        Project filterUserProject = project;
        for (AbstractFacade.FilterBy filterBy : filters) {
            if (filterBy.getParam().compareToIgnoreCase(Filters.ENDPOINT_ID.name()) == 0) {
                ExperimentsEndpointDTO endpoint = verifyExperimentsEndpoint(project, filterBy.getValue());
                selectedEndpoints.put(endpoint.getParentProject().getInode().getId(), endpoint);
            } else if (filterBy.getParam().compareToIgnoreCase(Filters.NAME_LIKE.name()) == 0) {
                provFilesParamBuilder.filterLikeXAttr(EXPERIMENT_SUMMARY_XATTR_NAME + ".name", filterBy.getValue());
            } else if (filterBy.getParam().compareToIgnoreCase(Filters.NAME_EQ.name()) == 0) {
                provFilesParamBuilder.filterByXAttr(EXPERIMENT_SUMMARY_XATTR_NAME + ".name", filterBy.getValue());
            } else if (filterBy.getParam().compareToIgnoreCase(Filters.DATE_START_LT.name()) == 0) {
                Long timestamp = getDate(filterBy.getField(), filterBy.getValue()).getTime();
                provFilesParamBuilder.filterByField(ProvStateParser.FieldsPF.CREATE_TIMESTAMP_LT, timestamp);
            } else if (filterBy.getParam().compareToIgnoreCase(Filters.DATE_START_GT.name()) == 0) {
                Long timestamp = getDate(filterBy.getField(), filterBy.getValue()).getTime();
                provFilesParamBuilder.filterByField(ProvStateParser.FieldsPF.CREATE_TIMESTAMP_GT, timestamp);
            } else if (filterBy.getParam().compareToIgnoreCase(Filters.USER.name()) == 0) {
                try {
                    filterUser = userFacade.find(Integer.parseInt(filterBy.getValue()));
                } catch (NumberFormatException e) {
                    throw new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.INFO, "expected int user id, found: " + filterBy.getValue());
                }
            } else if (filterBy.getParam().compareToIgnoreCase(Filters.USER_PROJECT.name()) == 0) {
                try {
                    filterUserProject = projectFacade.find(Integer.parseInt(filterBy.getValue()));
                } catch (NumberFormatException e) {
                    throw new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.INFO, "expected int user project id, found: " + filterBy.getValue());
                }
            } else if (filterBy.getParam().compareToIgnoreCase(Filters.STATE.name()) == 0) {
                provFilesParamBuilder.filterLikeXAttr(EXPERIMENT_SUMMARY_XATTR_NAME + ".state", filterBy.getValue());
            } else if (filterBy.getParam().compareToIgnoreCase(Filters.ID_EQ.name()) == 0) {
                provFilesParamBuilder.filterByXAttr(EXPERIMENT_SUMMARY_XATTR_NAME + ".id", filterBy.getValue());
            } else {
                throw new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.INFO, "Filter by - found: " + filterBy.getParam() + " expected:" + EnumSet.allOf(Filters.class));
            }
        }
        if (filterUser != null) {
            ProjectTeam member = projectTeamFacade.findByPrimaryKey(filterUserProject, filterUser);
            if (member == null) {
                throw new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.INFO, "Selected user: " + filterUser.getUid() + " is not part of project:" + filterUserProject.getId());
            }
            String hdfsUserStr = hdfsUsersController.getHdfsUserName(filterUserProject, filterUser);
            HdfsUsers hdfsUsers = hdfsUsersFacade.findByName(hdfsUserStr);
            provFilesParamBuilder.filterByField(ProvStateParser.FieldsP.USER_ID, hdfsUsers.getId().toString());
        }
    }
    // an endpoint always has to be selected, if none provided, then all accessible endpoints are used
    if (selectedEndpoints.isEmpty()) {
        for (ExperimentsEndpointDTO endpoint : experimentsController.getExperimentsEndpoints(project)) {
            selectedEndpoints.put(endpoint.getParentProject().getInode().getId(), endpoint);
        }
    }
    for (ExperimentsEndpointDTO endpoint : selectedEndpoints.values()) {
        provFilesParamBuilder.filterByField(ProvStateParser.FieldsP.PROJECT_I_ID, endpoint.getParentProject().getInode().getId()).filterByField(ProvStateParser.FieldsP.DATASET_I_ID, endpoint.getDatasetInodeId());
    }
    return Pair.with(provFilesParamBuilder, selectedEndpoints);
}
Also used : HashMap(java.util.HashMap) HdfsUsers(io.hops.hopsworks.persistence.entity.hdfs.user.HdfsUsers) Users(io.hops.hopsworks.persistence.entity.user.Users) GenericException(io.hops.hopsworks.exceptions.GenericException) HdfsUsers(io.hops.hopsworks.persistence.entity.hdfs.user.HdfsUsers) ProvStateParamBuilder(io.hops.hopsworks.common.provenance.state.ProvStateParamBuilder) Project(io.hops.hopsworks.persistence.entity.project.Project) ProjectTeam(io.hops.hopsworks.persistence.entity.project.team.ProjectTeam) AbstractFacade(io.hops.hopsworks.common.dao.AbstractFacade) ExperimentsEndpointDTO(io.hops.hopsworks.api.experiments.dto.ExperimentsEndpointDTO)

Example 4 with ProvStateParamBuilder

use of io.hops.hopsworks.common.provenance.state.ProvStateParamBuilder in project hopsworks by logicalclocks.

the class ModelsBuilder method build.

// Build collection
public ModelDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, Users user, Project userProject, Project modelRegistryProject) throws ModelRegistryException, GenericException, SchematizedTagException, MetadataException {
    ModelDTO dto = new ModelDTO();
    uri(dto, uriInfo, userProject, modelRegistryProject);
    expand(dto, resourceRequest);
    dto.setCount(0l);
    if (dto.isExpand()) {
        validatePagination(resourceRequest);
        ProvStateDTO fileState;
        try {
            Pair<ProvStateParamBuilder, ModelRegistryDTO> provFilesParamBuilder = buildModelProvenanceParams(userProject, modelRegistryProject, resourceRequest);
            if (provFilesParamBuilder.getValue1() == null) {
                // no endpoint - no results
                return dto;
            }
            fileState = provenanceController.provFileStateList(provFilesParamBuilder.getValue1().getParentProject(), provFilesParamBuilder.getValue0());
            List<ProvStateDTO> models = new LinkedList<>(fileState.getItems());
            dto.setCount(fileState.getCount());
            String modelsDatasetPath = modelUtils.getModelsDatasetPath(userProject, modelRegistryProject);
            for (ProvStateDTO fileProvStateHit : models) {
                ModelDTO modelDTO = build(uriInfo, resourceRequest, user, userProject, modelRegistryProject, fileProvStateHit, modelsDatasetPath);
                if (modelDTO != null) {
                    dto.addItem(modelDTO);
                }
            }
        } catch (ProvenanceException e) {
            if (ProvHelper.missingMappingForField(e)) {
                LOGGER.log(Level.WARNING, "Could not find elastic mapping for experiments query", e);
                return dto;
            } else {
                throw new ModelRegistryException(RESTCodes.ModelRegistryErrorCode.MODEL_LIST_FAILED, Level.FINE, "Unable to list models for project " + modelRegistryProject.getName(), e.getMessage(), e);
            }
        } catch (DatasetException e) {
            throw new ModelRegistryException(RESTCodes.ModelRegistryErrorCode.MODEL_LIST_FAILED, Level.FINE, "Unable to list models for project " + modelRegistryProject.getName(), e.getMessage(), e);
        }
    }
    return dto;
}
Also used : ProvStateParamBuilder(io.hops.hopsworks.common.provenance.state.ProvStateParamBuilder) ModelDTO(io.hops.hopsworks.api.modelregistry.models.dto.ModelDTO) ProvenanceException(io.hops.hopsworks.exceptions.ProvenanceException) ModelRegistryDTO(io.hops.hopsworks.api.modelregistry.dto.ModelRegistryDTO) ProvStateDTO(io.hops.hopsworks.common.provenance.state.dto.ProvStateDTO) LinkedList(java.util.LinkedList) ModelRegistryException(io.hops.hopsworks.exceptions.ModelRegistryException) DatasetException(io.hops.hopsworks.exceptions.DatasetException)

Example 5 with ProvStateParamBuilder

use of io.hops.hopsworks.common.provenance.state.ProvStateParamBuilder in project hopsworks by logicalclocks.

the class ExperimentsBuilder method build.

// Build collection
public ExperimentDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, Project project, Users user) throws ExperimentsException {
    ExperimentDTO dto = new ExperimentDTO();
    uri(dto, uriInfo, project);
    expand(dto, resourceRequest);
    dto.setCount(0l);
    validatePagination(resourceRequest);
    if (dto.isExpand()) {
        try {
            Pair<ProvStateParamBuilder, Map<Long, ExperimentsEndpointDTO>> provFilesParamBuilder = buildExperimentProvenanceParams(project, resourceRequest);
            if (provFilesParamBuilder.getValue1().isEmpty()) {
                // no endpoint - no results
                return dto;
            }
            ProvStateDTO fileState = provenanceController.provFileStateList(project, provFilesParamBuilder.getValue0());
            if (fileState != null) {
                List<ProvStateDTO> experiments = fileState.getItems();
                dto.setCount(fileState.getCount());
                if (experiments != null && !experiments.isEmpty()) {
                    for (ProvStateDTO fileProvStateHit : experiments) {
                        ExperimentDTO experimentDTO = build(uriInfo, resourceRequest, project, user, provFilesParamBuilder.getValue1(), fileProvStateHit);
                        if (experimentDTO != null) {
                            dto.addItem(experimentDTO);
                        }
                    }
                }
            }
        } catch (ExperimentsException | DatasetException | ProvenanceException | MetadataException | GenericException e) {
            if (e instanceof ProvenanceException && ProvHelper.missingMappingForField((ProvenanceException) e)) {
                LOGGER.log(Level.WARNING, "Could not find elastic mapping for experiments query", e);
                return dto;
            } else {
                throw new ExperimentsException(RESTCodes.ExperimentsErrorCode.EXPERIMENT_LIST_FAILED, Level.FINE, "Unable to list experiments for project " + project.getName(), e.getMessage(), e);
            }
        }
    }
    return dto;
}
Also used : ProvStateParamBuilder(io.hops.hopsworks.common.provenance.state.ProvStateParamBuilder) ProvenanceException(io.hops.hopsworks.exceptions.ProvenanceException) ExperimentsException(io.hops.hopsworks.exceptions.ExperimentsException) ExperimentDTO(io.hops.hopsworks.api.experiments.dto.ExperimentDTO) Map(java.util.Map) HashMap(java.util.HashMap) ProvStateDTO(io.hops.hopsworks.common.provenance.state.dto.ProvStateDTO) GenericException(io.hops.hopsworks.exceptions.GenericException) MetadataException(io.hops.hopsworks.exceptions.MetadataException) DatasetException(io.hops.hopsworks.exceptions.DatasetException)

Aggregations

ProvStateParamBuilder (io.hops.hopsworks.common.provenance.state.ProvStateParamBuilder)6 ProvStateDTO (io.hops.hopsworks.common.provenance.state.dto.ProvStateDTO)4 GenericException (io.hops.hopsworks.exceptions.GenericException)3 ModelRegistryDTO (io.hops.hopsworks.api.modelregistry.dto.ModelRegistryDTO)2 AbstractFacade (io.hops.hopsworks.common.dao.AbstractFacade)2 DatasetException (io.hops.hopsworks.exceptions.DatasetException)2 ProvenanceException (io.hops.hopsworks.exceptions.ProvenanceException)2 HdfsUsers (io.hops.hopsworks.persistence.entity.hdfs.user.HdfsUsers)2 Project (io.hops.hopsworks.persistence.entity.project.Project)2 ProjectTeam (io.hops.hopsworks.persistence.entity.project.team.ProjectTeam)2 Users (io.hops.hopsworks.persistence.entity.user.Users)2 HashMap (java.util.HashMap)2 ExperimentDTO (io.hops.hopsworks.api.experiments.dto.ExperimentDTO)1 ExperimentsEndpointDTO (io.hops.hopsworks.api.experiments.dto.ExperimentsEndpointDTO)1 ModelDTO (io.hops.hopsworks.api.modelregistry.models.dto.ModelDTO)1 ExperimentsException (io.hops.hopsworks.exceptions.ExperimentsException)1 MetadataException (io.hops.hopsworks.exceptions.MetadataException)1 ModelRegistryException (io.hops.hopsworks.exceptions.ModelRegistryException)1 LinkedList (java.util.LinkedList)1 Map (java.util.Map)1