Examples with DatasetVersion edu.harvard.iq.dataverse.DatasetVersion used on opensource projects

Example 56 with DatasetVersion

use of edu.harvard.iq.dataverse.DatasetVersion in project dataverse by IQSS.

the class SolrIndexServiceBean method filesToReIndexPermissionsFor.

private List<DataFile> filesToReIndexPermissionsFor(Dataset dataset) {
    List<DataFile> filesToReindexPermissionsFor = new ArrayList<>();
    Map<DatasetVersion.VersionState, Boolean> desiredCards = searchPermissionsService.getDesiredCards(dataset);
    for (DatasetVersion version : datasetVersionsToBuildCardsFor(dataset)) {
        boolean cardShouldExist = desiredCards.get(version.getVersionState());
        if (cardShouldExist) {
            for (FileMetadata fileMetadata : version.getFileMetadatas()) {
                filesToReindexPermissionsFor.add(fileMetadata.getDataFile());
            }
        }
    }
    return filesToReindexPermissionsFor;
}

Example 57 with DatasetVersion

use of edu.harvard.iq.dataverse.DatasetVersion in project dataverse by IQSS.

the class SolrIndexServiceBean method constructDatasetSolrDocs.

private List<DvObjectSolrDoc> constructDatasetSolrDocs(Dataset dataset) {
    List<DvObjectSolrDoc> emptyList = new ArrayList<>();
    List<DvObjectSolrDoc> solrDocs = emptyList;
    Map<DatasetVersion.VersionState, Boolean> desiredCards = searchPermissionsService.getDesiredCards(dataset);
    for (DatasetVersion version : datasetVersionsToBuildCardsFor(dataset)) {
        boolean cardShouldExist = desiredCards.get(version.getVersionState());
        if (cardShouldExist) {
            DvObjectSolrDoc datasetSolrDoc = makeDatasetSolrDoc(version);
            solrDocs.add(datasetSolrDoc);
        }
    }
    return solrDocs;
}

Example 58 with DatasetVersion

use of edu.harvard.iq.dataverse.DatasetVersion in project dataverse by IQSS.

the class SolrIndexServiceBean method constructDatafileSolrDocsFromDataset.

private List<DvObjectSolrDoc> constructDatafileSolrDocsFromDataset(Dataset dataset) {
    List<DvObjectSolrDoc> datafileSolrDocs = new ArrayList<>();
    Map<DatasetVersion.VersionState, Boolean> desiredCards = searchPermissionsService.getDesiredCards(dataset);
    for (DatasetVersion datasetVersionFileIsAttachedTo : datasetVersionsToBuildCardsFor(dataset)) {
        boolean cardShouldExist = desiredCards.get(datasetVersionFileIsAttachedTo.getVersionState());
        if (cardShouldExist) {
            List<String> perms = new ArrayList<>();
            if (unpublishedDataRelatedToMeModeEnabled) {
                if (datasetVersionFileIsAttachedTo.isReleased()) {
                    perms.add(IndexServiceBean.getPublicGroupString());
                } else {
                    perms = searchPermissionsService.findDatasetVersionPerms(datasetVersionFileIsAttachedTo);
                }
            } else {
                perms = searchPermissionsService.findDatasetVersionPerms(datasetVersionFileIsAttachedTo);
            }
            for (FileMetadata fileMetadata : datasetVersionFileIsAttachedTo.getFileMetadatas()) {
                Long fileId = fileMetadata.getDataFile().getId();
                String solrIdStart = IndexServiceBean.solrDocIdentifierFile + fileId;
                String solrIdEnd = getDatasetOrDataFileSolrEnding(datasetVersionFileIsAttachedTo.getVersionState());
                String solrId = solrIdStart + solrIdEnd;
                DvObjectSolrDoc dataFileSolrDoc = new DvObjectSolrDoc(fileId.toString(), solrId, datasetVersionFileIsAttachedTo.getId(), fileMetadata.getLabel(), perms);
                logger.fine("adding fileid " + fileId);
                datafileSolrDocs.add(dataFileSolrDoc);
            }
        }
    }
    return datafileSolrDocs;
}

Example 59 with DatasetVersion

use of edu.harvard.iq.dataverse.DatasetVersion in project dataverse by IQSS.

the class SolrIndexServiceBean method constructDatafileSolrDocs.

// private List<DvObjectSolrDoc> constructDatafileSolrDocs(DataFile dataFile) {
private List<DvObjectSolrDoc> constructDatafileSolrDocs(DataFile dataFile, Map<Long, List<String>> permStringByDatasetVersion) {
    List<DvObjectSolrDoc> datafileSolrDocs = new ArrayList<>();
    Map<DatasetVersion.VersionState, Boolean> desiredCards = searchPermissionsService.getDesiredCards(dataFile.getOwner());
    for (DatasetVersion datasetVersionFileIsAttachedTo : datasetVersionsToBuildCardsFor(dataFile.getOwner())) {
        boolean cardShouldExist = desiredCards.get(datasetVersionFileIsAttachedTo.getVersionState());
        if (cardShouldExist) {
            String solrIdStart = IndexServiceBean.solrDocIdentifierFile + dataFile.getId();
            String solrIdEnd = getDatasetOrDataFileSolrEnding(datasetVersionFileIsAttachedTo.getVersionState());
            String solrId = solrIdStart + solrIdEnd;
            List<String> perms = new ArrayList<>();
            if (unpublishedDataRelatedToMeModeEnabled) {
                List<String> cachedPerms = null;
                if (permStringByDatasetVersion != null) {
                    cachedPerms = permStringByDatasetVersion.get(datasetVersionFileIsAttachedTo.getId());
                }
                if (cachedPerms != null) {
                    logger.fine("reusing cached perms for file " + dataFile.getId());
                    perms = cachedPerms;
                } else if (datasetVersionFileIsAttachedTo.isReleased()) {
                    logger.fine("no cached perms, file is public/discoverable/searchable for file " + dataFile.getId());
                    perms.add(IndexServiceBean.getPublicGroupString());
                } else {
                    // go to the well (slow)
                    logger.fine("no cached perms, file is not public, finding perms for file " + dataFile.getId());
                    perms = searchPermissionsService.findDatasetVersionPerms(datasetVersionFileIsAttachedTo);
                }
            } else {
                // This should never be executed per the deprecation notice on the boolean.
                perms = searchPermissionsService.findDatasetVersionPerms(datasetVersionFileIsAttachedTo);
            }
            DvObjectSolrDoc dataFileSolrDoc = new DvObjectSolrDoc(dataFile.getId().toString(), solrId, datasetVersionFileIsAttachedTo.getId(), dataFile.getDisplayName(), perms);
            datafileSolrDocs.add(dataFileSolrDoc);
        }
    }
    return datafileSolrDocs;
}

Example 60 with DatasetVersion

use of edu.harvard.iq.dataverse.DatasetVersion in project dataverse by IQSS.

the class IndexServiceBean method indexDataset.

public Future<String> indexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) {
    logger.fine("indexing dataset " + dataset.getId());
    /**
     * @todo should we use solrDocIdentifierDataset or
     * IndexableObject.IndexableTypes.DATASET.getName() + "_" ?
     */
    // String solrIdPublished = solrDocIdentifierDataset + dataset.getId();
    String solrIdPublished = determinePublishedDatasetSolrDocId(dataset);
    String solrIdDraftDataset = IndexableObject.IndexableTypes.DATASET.getName() + "_" + dataset.getId() + IndexableDataset.DatasetState.WORKING_COPY.getSuffix();
    // String solrIdDeaccessioned = IndexableObject.IndexableTypes.DATASET.getName() + "_" + dataset.getId() + IndexableDataset.DatasetState.DEACCESSIONED.getSuffix();
    String solrIdDeaccessioned = determineDeaccessionedDatasetId(dataset);
    StringBuilder debug = new StringBuilder();
    debug.append("\ndebug:\n");
    int numPublishedVersions = 0;
    List<DatasetVersion> versions = dataset.getVersions();
    List<String> solrIdsOfFilesToDelete = new ArrayList<>();
    for (DatasetVersion datasetVersion : versions) {
        Long versionDatabaseId = datasetVersion.getId();
        String versionTitle = datasetVersion.getTitle();
        String semanticVersion = datasetVersion.getSemanticVersion();
        DatasetVersion.VersionState versionState = datasetVersion.getVersionState();
        if (versionState.equals(DatasetVersion.VersionState.RELEASED)) {
            numPublishedVersions += 1;
        }
        debug.append("version found with database id " + versionDatabaseId + "\n");
        debug.append("- title: " + versionTitle + "\n");
        debug.append("- semanticVersion-VersionState: " + semanticVersion + "-" + versionState + "\n");
        List<FileMetadata> fileMetadatas = datasetVersion.getFileMetadatas();
        List<String> fileInfo = new ArrayList<>();
        for (FileMetadata fileMetadata : fileMetadatas) {
            String solrIdOfPublishedFile = solrDocIdentifierFile + fileMetadata.getDataFile().getId();
            /**
             * It sounds weird but the first thing we'll do is preemptively
             * delete the Solr documents of all published files. Don't
             * worry, published files will be re-indexed later along with
             * the dataset. We do this so users can delete files from
             * published versions of datasets and then re-publish a new
             * version without fear that their old published files (now
             * deleted from the latest published version) will be
             * searchable. See also
             * https://github.com/IQSS/dataverse/issues/762
             */
            solrIdsOfFilesToDelete.add(solrIdOfPublishedFile);
            fileInfo.add(fileMetadata.getDataFile().getId() + ":" + fileMetadata.getLabel());
        }
        try {
            /**
             * Preemptively delete *all* Solr documents for files associated
             * with the dataset based on a Solr query.
             *
             * We must query Solr for this information because the file has
             * been deleted from the database ( perhaps when Solr was down,
             * as reported in https://github.com/IQSS/dataverse/issues/2086
             * ) so the database doesn't even know about the file. It's an
             * orphan.
             *
             * @todo This Solr query should make the iteration above based
             * on the database unnecessary because it the Solr query should
             * find all files for the dataset. We can probably remove the
             * iteration above after an "index all" has been performed.
             * Without an "index all" we won't be able to find files based
             * on parentId because that field wasn't searchable in 4.0.
             *
             * @todo We should also delete the corresponding Solr
             * "permission" documents for the files.
             */
            List<String> allFilesForDataset = findFilesOfParentDataset(dataset.getId());
            solrIdsOfFilesToDelete.addAll(allFilesForDataset);
        } catch (SearchException | NullPointerException ex) {
            logger.fine("could not run search of files to delete: " + ex);
        }
        int numFiles = 0;
        if (fileMetadatas != null) {
            numFiles = fileMetadatas.size();
        }
        debug.append("- files: " + numFiles + " " + fileInfo.toString() + "\n");
    }
    debug.append("numPublishedVersions: " + numPublishedVersions + "\n");
    if (doNormalSolrDocCleanUp) {
        IndexResponse resultOfAttemptToPremptivelyDeletePublishedFiles = solrIndexService.deleteMultipleSolrIds(solrIdsOfFilesToDelete);
        debug.append("result of attempt to premptively deleted published files before reindexing: " + resultOfAttemptToPremptivelyDeletePublishedFiles + "\n");
    }
    DatasetVersion latestVersion = dataset.getLatestVersion();
    String latestVersionStateString = latestVersion.getVersionState().name();
    DatasetVersion.VersionState latestVersionState = latestVersion.getVersionState();
    DatasetVersion releasedVersion = dataset.getReleasedVersion();
    boolean atLeastOnePublishedVersion = false;
    if (releasedVersion != null) {
        atLeastOnePublishedVersion = true;
    } else {
        atLeastOnePublishedVersion = false;
    }
    Map<DatasetVersion.VersionState, Boolean> desiredCards = new LinkedHashMap<>();
    /**
     * @todo refactor all of this below and have a single method that takes
     * the map of desired cards (which correspond to Solr documents) as one
     * of the arguments and does all the operations necessary to achieve the
     * desired state.
     */
    StringBuilder results = new StringBuilder();
    if (atLeastOnePublishedVersion == false) {
        results.append("No published version, nothing will be indexed as ").append(solrIdPublished).append("\n");
        if (latestVersionState.equals(DatasetVersion.VersionState.DRAFT)) {
            desiredCards.put(DatasetVersion.VersionState.DRAFT, true);
            IndexableDataset indexableDraftVersion = new IndexableDataset(latestVersion);
            String indexDraftResult = addOrUpdateDataset(indexableDraftVersion);
            results.append("The latest version is a working copy (latestVersionState: ").append(latestVersionStateString).append(") and indexing was attempted for ").append(solrIdDraftDataset).append(" (limited discoverability). Result: ").append(indexDraftResult).append("\n");
            desiredCards.put(DatasetVersion.VersionState.DEACCESSIONED, false);
            if (doNormalSolrDocCleanUp) {
                String deleteDeaccessionedResult = removeDeaccessioned(dataset);
                results.append("Draft exists, no need for deaccessioned version. Deletion attempted for ").append(solrIdDeaccessioned).append(" (and files). Result: ").append(deleteDeaccessionedResult).append("\n");
            }
            desiredCards.put(DatasetVersion.VersionState.RELEASED, false);
            if (doNormalSolrDocCleanUp) {
                String deletePublishedResults = removePublished(dataset);
                results.append("No published version. Attempting to delete traces of published version from index. Result: ").append(deletePublishedResults).append("\n");
            }
            /**
             * Desired state for existence of cards: {DRAFT=true,
             * DEACCESSIONED=false, RELEASED=false}
             *
             * No published version, nothing will be indexed as dataset_17
             *
             * The latest version is a working copy (latestVersionState:
             * DRAFT) and indexing was attempted for dataset_17_draft
             * (limited discoverability). Result: indexed dataset 17 as
             * dataset_17_draft. filesIndexed: [datafile_18_draft]
             *
             * Draft exists, no need for deaccessioned version. Deletion
             * attempted for dataset_17_deaccessioned (and files). Result:
             * Attempted to delete dataset_17_deaccessioned from Solr index.
             * updateReponse was:
             * {responseHeader={status=0,QTime=1}}Attempted to delete
             * datafile_18_deaccessioned from Solr index. updateReponse was:
             * {responseHeader={status=0,QTime=1}}
             *
             * No published version. Attempting to delete traces of
             * published version from index. Result: Attempted to delete
             * dataset_17 from Solr index. updateReponse was:
             * {responseHeader={status=0,QTime=1}}Attempted to delete
             * datafile_18 from Solr index. updateReponse was:
             * {responseHeader={status=0,QTime=0}}
             */
            String result = getDesiredCardState(desiredCards) + results.toString() + debug.toString();
            logger.fine(result);
            indexDatasetPermissions(dataset);
            return new AsyncResult<>(result);
        } else if (latestVersionState.equals(DatasetVersion.VersionState.DEACCESSIONED)) {
            desiredCards.put(DatasetVersion.VersionState.DEACCESSIONED, true);
            IndexableDataset indexableDeaccessionedVersion = new IndexableDataset(latestVersion);
            String indexDeaccessionedVersionResult = addOrUpdateDataset(indexableDeaccessionedVersion);
            results.append("No draft version. Attempting to index as deaccessioned. Result: ").append(indexDeaccessionedVersionResult).append("\n");
            desiredCards.put(DatasetVersion.VersionState.RELEASED, false);
            if (doNormalSolrDocCleanUp) {
                String deletePublishedResults = removePublished(dataset);
                results.append("No published version. Attempting to delete traces of published version from index. Result: ").append(deletePublishedResults).append("\n");
            }
            desiredCards.put(DatasetVersion.VersionState.DRAFT, false);
            if (doNormalSolrDocCleanUp) {
                List<String> solrDocIdsForDraftFilesToDelete = findSolrDocIdsForDraftFilesToDelete(dataset);
                String deleteDraftDatasetVersionResult = removeSolrDocFromIndex(solrIdDraftDataset);
                String deleteDraftFilesResults = deleteDraftFiles(solrDocIdsForDraftFilesToDelete);
                results.append("Attempting to delete traces of drafts. Result: ").append(deleteDraftDatasetVersionResult).append(deleteDraftFilesResults).append("\n");
            }
            /**
             * Desired state for existence of cards: {DEACCESSIONED=true,
             * RELEASED=false, DRAFT=false}
             *
             * No published version, nothing will be indexed as dataset_17
             *
             * No draft version. Attempting to index as deaccessioned.
             * Result: indexed dataset 17 as dataset_17_deaccessioned.
             * filesIndexed: []
             *
             * No published version. Attempting to delete traces of
             * published version from index. Result: Attempted to delete
             * dataset_17 from Solr index. updateReponse was:
             * {responseHeader={status=0,QTime=0}}Attempted to delete
             * datafile_18 from Solr index. updateReponse was:
             * {responseHeader={status=0,QTime=3}}
             *
             * Attempting to delete traces of drafts. Result: Attempted to
             * delete dataset_17_draft from Solr index. updateReponse was:
             * {responseHeader={status=0,QTime=1}}
             */
            String result = getDesiredCardState(desiredCards) + results.toString() + debug.toString();
            logger.fine(result);
            indexDatasetPermissions(dataset);
            return new AsyncResult<>(result);
        } else {
            String result = "No-op. Unexpected condition reached: No released version and latest version is neither draft nor deaccessioned";
            logger.fine(result);
            return new AsyncResult<>(result);
        }
    } else if (atLeastOnePublishedVersion == true) {
        results.append("Published versions found. ").append("Will attempt to index as ").append(solrIdPublished).append(" (discoverable by anonymous)\n");
        if (latestVersionState.equals(DatasetVersion.VersionState.RELEASED) || latestVersionState.equals(DatasetVersion.VersionState.DEACCESSIONED)) {
            desiredCards.put(DatasetVersion.VersionState.RELEASED, true);
            IndexableDataset indexableReleasedVersion = new IndexableDataset(releasedVersion);
            String indexReleasedVersionResult = addOrUpdateDataset(indexableReleasedVersion);
            results.append("Attempted to index " + solrIdPublished).append(". Result: ").append(indexReleasedVersionResult).append("\n");
            desiredCards.put(DatasetVersion.VersionState.DRAFT, false);
            if (doNormalSolrDocCleanUp) {
                List<String> solrDocIdsForDraftFilesToDelete = findSolrDocIdsForDraftFilesToDelete(dataset);
                String deleteDraftDatasetVersionResult = removeSolrDocFromIndex(solrIdDraftDataset);
                String deleteDraftFilesResults = deleteDraftFiles(solrDocIdsForDraftFilesToDelete);
                results.append("The latest version is published. Attempting to delete drafts. Result: ").append(deleteDraftDatasetVersionResult).append(deleteDraftFilesResults).append("\n");
            }
            desiredCards.put(DatasetVersion.VersionState.DEACCESSIONED, false);
            if (doNormalSolrDocCleanUp) {
                String deleteDeaccessionedResult = removeDeaccessioned(dataset);
                results.append("No need for deaccessioned version. Deletion attempted for ").append(solrIdDeaccessioned).append(". Result: ").append(deleteDeaccessionedResult);
            }
            /**
             * Desired state for existence of cards: {RELEASED=true,
             * DRAFT=false, DEACCESSIONED=false}
             *
             * Released versions found: 1. Will attempt to index as
             * dataset_17 (discoverable by anonymous)
             *
             * Attempted to index dataset_17. Result: indexed dataset 17 as
             * dataset_17. filesIndexed: [datafile_18]
             *
             * The latest version is published. Attempting to delete drafts.
             * Result: Attempted to delete dataset_17_draft from Solr index.
             * updateReponse was: {responseHeader={status=0,QTime=1}}
             *
             * No need for deaccessioned version. Deletion attempted for
             * dataset_17_deaccessioned. Result: Attempted to delete
             * dataset_17_deaccessioned from Solr index. updateReponse was:
             * {responseHeader={status=0,QTime=1}}Attempted to delete
             * datafile_18_deaccessioned from Solr index. updateReponse was:
             * {responseHeader={status=0,QTime=0}}
             */
            String result = getDesiredCardState(desiredCards) + results.toString() + debug.toString();
            logger.fine(result);
            indexDatasetPermissions(dataset);
            return new AsyncResult<>(result);
        } else if (latestVersionState.equals(DatasetVersion.VersionState.DRAFT)) {
            IndexableDataset indexableDraftVersion = new IndexableDataset(latestVersion);
            desiredCards.put(DatasetVersion.VersionState.DRAFT, true);
            String indexDraftResult = addOrUpdateDataset(indexableDraftVersion);
            results.append("The latest version is a working copy (latestVersionState: ").append(latestVersionStateString).append(") and will be indexed as ").append(solrIdDraftDataset).append(" (limited visibility). Result: ").append(indexDraftResult).append("\n");
            desiredCards.put(DatasetVersion.VersionState.RELEASED, true);
            IndexableDataset indexableReleasedVersion = new IndexableDataset(releasedVersion);
            String indexReleasedVersionResult = addOrUpdateDataset(indexableReleasedVersion);
            results.append("There is a published version we will attempt to index. Result: ").append(indexReleasedVersionResult).append("\n");
            desiredCards.put(DatasetVersion.VersionState.DEACCESSIONED, false);
            if (doNormalSolrDocCleanUp) {
                String deleteDeaccessionedResult = removeDeaccessioned(dataset);
                results.append("No need for deaccessioned version. Deletion attempted for ").append(solrIdDeaccessioned).append(". Result: ").append(deleteDeaccessionedResult);
            }
            /**
             * Desired state for existence of cards: {DRAFT=true,
             * RELEASED=true, DEACCESSIONED=false}
             *
             * Released versions found: 1. Will attempt to index as
             * dataset_17 (discoverable by anonymous)
             *
             * The latest version is a working copy (latestVersionState:
             * DRAFT) and will be indexed as dataset_17_draft (limited
             * visibility). Result: indexed dataset 17 as dataset_17_draft.
             * filesIndexed: [datafile_18_draft]
             *
             * There is a published version we will attempt to index.
             * Result: indexed dataset 17 as dataset_17. filesIndexed:
             * [datafile_18]
             *
             * No need for deaccessioned version. Deletion attempted for
             * dataset_17_deaccessioned. Result: Attempted to delete
             * dataset_17_deaccessioned from Solr index. updateReponse was:
             * {responseHeader={status=0,QTime=1}}Attempted to delete
             * datafile_18_deaccessioned from Solr index. updateReponse was:
             * {responseHeader={status=0,QTime=0}}
             */
            String result = getDesiredCardState(desiredCards) + results.toString() + debug.toString();
            logger.fine(result);
            indexDatasetPermissions(dataset);
            return new AsyncResult<>(result);
        } else {
            String result = "No-op. Unexpected condition reached: There is at least one published version but the latest version is neither published nor draft";
            logger.fine(result);
            return new AsyncResult<>(result);
        }
    } else {
        String result = "No-op. Unexpected condition reached: Has a version been published or not?";
        logger.fine(result);
        return new AsyncResult<>(result);
    }
}