Examples with ProvenanceException io.hops.hopsworks.exceptions.ProvenanceException used on opensource projects

Example 6 with ProvenanceException

use of io.hops.hopsworks.exceptions.ProvenanceException in project hopsworks by logicalclocks.

the class LibraryController method pipList.

/**
 * @param <R> parsed elastic item
 * @param <S1> intermediate result wrapped in Try
 * @param <S2> final result
 * @return
 * @throws ProvenanceException
 */
private <R, S1, S2> S2 pipList(String query, Integer offset, Integer limit, LibraryController.HandlerFactory<R, S1, S2> handlerFactory, String index) throws ServiceException {
    Pair<Long, Try<S1>> searchResult;
    try {
        CheckedSupplier<SearchRequest, ProvenanceException> srF = ElasticHelper.baseSearchRequest(index, settings.getElasticDefaultScrollPageSize()).andThen(ElasticHelper.withPagination(offset, limit, settings.getElasticDefaultScrollPageSize()));
        SearchRequest request = srF.get();
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        sourceBuilder.query(ElasticHelper.fullTextSearch("library", query));
        request.source(sourceBuilder);
        searchResult = provElasticController.search(request, handlerFactory.getHandler());
    } catch (ElasticException | ProvenanceException pe) {
        throw new ServiceException(RESTCodes.ServiceErrorCode.ANACONDA_LIST_LIB_ERROR, Level.FINE, "Unable to list python libraries", pe.getMessage(), pe);
    }
    return handlerFactory.checkedResult(searchResult);
}

Example 7 with ProvenanceException

use of io.hops.hopsworks.exceptions.ProvenanceException in project hopsworks by logicalclocks.

the class ProvAppController method provAppState.

public Map<String, Map<Provenance.AppState, ProvAppStateElastic>> provAppState(Map<ProvParser.Field, ProvParser.FilterVal> filterBy, List<Pair<ProvParser.Field, SortOrder>> sortBy, Integer offset, Integer limit) throws ProvenanceException {
    CheckedSupplier<SearchRequest, ProvenanceException> srF = ElasticHelper.scrollingSearchRequest(Settings.ELASTIC_INDEX_APP_PROVENANCE, settings.getElasticDefaultScrollPageSize()).andThen(provAppStateQB(filterBy)).andThen(ElasticHelper.sortBy(sortBy)).andThen(ElasticHelper.withPagination(offset, limit, settings.getElasticMaxScrollPageSize()));
    SearchRequest request = srF.get();
    Pair<Long, Try<ElasticAppStatesObj>> searchResult;
    try {
        searchResult = client.searchScrolling(request, ElasticAppStatesObj.getHandler());
    } catch (ElasticException e) {
        String msg = "provenance - elastic query problem";
        throw ProvHelper.fromElastic(e, msg, msg + " - app state");
    }
    return ElasticAppStatesObj.checkedResult(searchResult);
}

Example 8 with ProvenanceException

use of io.hops.hopsworks.exceptions.ProvenanceException in project hopsworks by logicalclocks.

the class HopsFSProvenanceController method setProvCoreXAttr.

private void setProvCoreXAttr(String path, ProvCoreDTO provCore, DistributedFileSystemOps udfso) throws ProvenanceException {
    try {
        String provType = converter.marshal(provCore);
        xattrCtrl.upsertProvXAttr(udfso, path, ProvXAttrs.PROV_XATTR_CORE_VAL, provType.getBytes());
    } catch (GenericException | DatasetException | MetadataException e) {
        throw new ProvenanceException(RESTCodes.ProvenanceErrorCode.FS_ERROR, Level.WARNING, "hopsfs - set xattr - prov core - error", "hopsfs - set xattr - prov core - error", e);
    }
}

Example 9 with ProvenanceException

use of io.hops.hopsworks.exceptions.ProvenanceException in project hopsworks by logicalclocks.

the class HopsFSProvenanceController method getDatasetsProvType.

public List<ProvDatasetDTO> getDatasetsProvType(Users user, Project project) throws ProvenanceException {
    String hdfsUsername = hdfsUsersController.getHdfsUserName(project, user);
    DistributedFileSystemOps udfso = dfs.getDfsOps(hdfsUsername);
    try {
        List<ProvDatasetDTO> result = new ArrayList<>();
        for (Dataset dataset : project.getDatasetCollection()) {
            String datasetPath = Utils.getFileSystemDatasetPath(dataset, settings);
            ProvCoreDTO provCore = getProvCoreXAttr(datasetPath, udfso);
            if (provCore == null) {
                throw new ProvenanceException(RESTCodes.ProvenanceErrorCode.INTERNAL_ERROR, Level.WARNING, "malformed dataset - provenance", "no provenance core xattr");
            }
            ProvDatasetDTO dsState = new ProvDatasetDTO(dataset.getName(), dataset.getInode().getId(), provCore.getType());
            result.add(dsState);
        }
        for (DatasetSharedWith dataset : project.getDatasetSharedWithCollection()) {
            String datasetPath = Utils.getFileSystemDatasetPath(dataset.getDataset(), settings);
            ProvCoreDTO provCore = getProvCoreXAttr(datasetPath, udfso);
            if (provCore == null) {
                throw new ProvenanceException(RESTCodes.ProvenanceErrorCode.INTERNAL_ERROR, Level.WARNING, "malformed dataset - provenance", "no provenance core xattr");
            }
            ProvDatasetDTO dsState = new ProvDatasetDTO(dataset.getDataset().getProject().getName() + "::" + dataset.getDataset().getName(), dataset.getDataset().getInode().getId(), provCore.getType());
            result.add(dsState);
        }
        return result;
    } finally {
        if (udfso != null) {
            dfs.closeDfsClient(udfso);
        }
    }
}

Example 10 with ProvenanceException

use of io.hops.hopsworks.exceptions.ProvenanceException in project hopsworks by logicalclocks.

the class ProvStateParser method instance.

private static ProvStateDTO instance(ProvStateDTO result, Map<String, Object> auxMap) throws ProvenanceException {
    try {
        result.setProjectInodeId(ProvHelper.extractElasticField(auxMap, ProvStateParser.FieldsP.PROJECT_I_ID));
        result.setInodeId(ProvHelper.extractElasticField(auxMap, ProvStateParser.FieldsP.FILE_I_ID));
        result.setAppId(ProvHelper.extractElasticField(auxMap, ProvStateParser.FieldsP.APP_ID));
        result.setUserId(ProvHelper.extractElasticField(auxMap, ProvStateParser.FieldsP.USER_ID));
        result.setInodeName(ProvHelper.extractElasticField(auxMap, ProvStateParser.FieldsP.FILE_NAME));
        result.setCreateTime(ProvHelper.extractElasticField(auxMap, ProvStateParser.FieldsP.CREATE_TIMESTAMP));
        result.setMlType(ProvHelper.extractElasticField(auxMap, ProvStateParser.FieldsP.ML_TYPE));
        result.setMlId(ProvHelper.extractElasticField(auxMap, ProvStateParser.FieldsP.ML_ID));
        result.setDatasetInodeId(ProvHelper.extractElasticField(auxMap, ProvStateParser.FieldsP.DATASET_I_ID));
        result.setParentInodeId(ProvHelper.extractElasticField(auxMap, ProvStateParser.FieldsP.PARENT_I_ID));
        result.setPartitionId(ProvHelper.extractElasticField(auxMap, ProvStateParser.FieldsP.PARTITION_ID));
        result.setProjectName(ProvHelper.extractElasticField(auxMap, ProvStateParser.FieldsP.PROJECT_NAME));
        result.setReadableCreateTime(ProvHelper.extractElasticField(auxMap, ProvStateParser.FieldsP.R_CREATE_TIMESTAMP));
        ProvHelper.extractElasticField(auxMap, ProvStateParser.FieldsP.ENTRY_TYPE);
        result.setXattrs(ProvHelper.extractElasticField(auxMap, ProvParser.XAttrField.XATTR_PROV, ProvHelper.asXAttrMap(), true));
        if (result.getXattrs() != null && result.getXattrs().containsKey(ProvStateParser.FieldsP.APP_ID.toString())) {
            // update if we have a value - useful if tls is disabled and out appId is set to none
            result.setAppId(ProvHelper.extractElasticField(result.getXattrs().get(ProvParser.Fields.APP_ID.toString())));
        }
        if (!auxMap.isEmpty()) {
            LOGGER.log(Level.FINE, "fields:{0} not managed in file state return", auxMap.keySet());
        }
    } catch (ClassCastException e) {
        String msg = "mismatch between DTO class and ProvSParser field types (elastic)";
        throw new ProvenanceException(RESTCodes.ProvenanceErrorCode.INTERNAL_ERROR, Level.WARNING, msg, msg, e);
    }
    return result;
}