Examples with DatasetFieldType edu.harvard.iq.dataverse.DatasetFieldType used on opensource projects

Example 16 with DatasetFieldType

use of edu.harvard.iq.dataverse.DatasetFieldType in project dataverse by IQSS.

the class JsonParser method convertKeywordsToSubjects.

/**
 * Special processing of keywords and subjects.  All keywords and subjects will be input
 * from foreign formats (DDI, dcterms, etc) as keywords.
 * As part of the parsing, we will move keywords that match subject controlled vocabulary values
 * into the subjects datasetField.
 * @param fields - the parsed datasetFields
 */
public void convertKeywordsToSubjects(List<DatasetField> fields) {
    DatasetField keywordField = null;
    for (DatasetField field : fields) {
        if (field.getDatasetFieldType().getName().equals("keyword")) {
            keywordField = field;
            break;
        }
    }
    if (keywordField == null) {
        // nothing to do.
        return;
    }
    DatasetFieldType type = datasetFieldSvc.findByNameOpt(DatasetFieldConstant.subject);
    // new list to hold subjects that we find
    List<ControlledVocabularyValue> subjects = new ArrayList<>();
    // Make new list to hold the non-subject keywords
    List<DatasetFieldCompoundValue> filteredValues = new ArrayList<>();
    for (DatasetFieldCompoundValue compoundVal : keywordField.getDatasetFieldCompoundValues()) {
        // Loop through the child fields to find the "keywordValue" field
        for (DatasetField childField : compoundVal.getChildDatasetFields()) {
            if (childField.getDatasetFieldType().getName().equals(DatasetFieldConstant.keywordValue)) {
                // check if this value is a subject
                ControlledVocabularyValue cvv = datasetFieldSvc.findControlledVocabularyValueByDatasetFieldTypeAndStrValue(type, childField.getValue(), lenient);
                if (cvv == null) {
                    // the keyword was not found in the subject list, so retain it in filtered list
                    filteredValues.add(compoundVal);
                } else {
                    // save the value for our subject field
                    if (!subjects.contains(cvv)) {
                        subjects.add(cvv);
                    }
                }
            }
        }
    }
    // if we have found any subjects in the keyword list, then update the keyword and subject fields appropriately.
    if (subjects.size() > 0) {
        keywordField.setDatasetFieldCompoundValues(filteredValues);
        DatasetField subjectField = new DatasetField();
        subjectField.setDatasetFieldType(type);
        for (ControlledVocabularyValue val : subjects) {
            int order = 0;
            val.setDisplayOrder(order);
            val.setDatasetFieldType(type);
            order++;
        }
        subjectField.setControlledVocabularyValues(subjects);
        fields.add(subjectField);
    }
}

Example 17 with DatasetFieldType

use of edu.harvard.iq.dataverse.DatasetFieldType in project dataverse by IQSS.

the class JsonPrinter method json.

public static JsonObjectBuilder json(MetadataBlock blk) {
    JsonObjectBuilder bld = jsonObjectBuilder();
    bld.add("id", blk.getId());
    bld.add("name", blk.getName());
    bld.add("displayName", blk.getDisplayName());
    JsonObjectBuilder fieldsBld = jsonObjectBuilder();
    for (DatasetFieldType df : new TreeSet<>(blk.getDatasetFieldTypes())) {
        fieldsBld.add(df.getName(), JsonPrinter.json(df));
    }
    bld.add("fields", fieldsBld);
    return bld;
}

Example 18 with DatasetFieldType

use of edu.harvard.iq.dataverse.DatasetFieldType in project dataverse by IQSS.

the class JsonPrinter method json.

public static JsonObjectBuilder json(DatasetFieldType fld) {
    JsonObjectBuilder fieldsBld = jsonObjectBuilder();
    fieldsBld.add("name", fld.getName());
    fieldsBld.add("displayName", fld.getDisplayName());
    fieldsBld.add("title", fld.getTitle());
    fieldsBld.add("type", fld.getFieldType().toString());
    fieldsBld.add("watermark", fld.getWatermark());
    fieldsBld.add("description", fld.getDescription());
    if (!fld.getChildDatasetFieldTypes().isEmpty()) {
        JsonObjectBuilder subFieldsBld = jsonObjectBuilder();
        for (DatasetFieldType subFld : fld.getChildDatasetFieldTypes()) {
            subFieldsBld.add(subFld.getName(), JsonPrinter.json(subFld));
        }
        fieldsBld.add("childFields", subFieldsBld);
    }
    return fieldsBld;
}

Example 19 with DatasetFieldType

use of edu.harvard.iq.dataverse.DatasetFieldType in project dataverse by IQSS.

the class CreateDataverseCommand method execute.

@Override
public Dataverse execute(CommandContext ctxt) throws CommandException {
    if (created.getOwner() == null) {
        if (ctxt.dataverses().isRootDataverseExists()) {
            throw new IllegalCommandException("Root Dataverse already exists. Cannot create another one", this);
        }
    }
    if (created.getCreateDate() == null) {
        created.setCreateDate(new Timestamp(new Date().getTime()));
    }
    if (created.getCreator() == null) {
        final User user = getRequest().getUser();
        if (user.isAuthenticated()) {
            created.setCreator((AuthenticatedUser) user);
        } else {
            throw new IllegalCommandException("Guest users cannot create a Dataverse.", this);
        }
    }
    if (created.getDataverseType() == null) {
        created.setDataverseType(Dataverse.DataverseType.UNCATEGORIZED);
    }
    if (created.getDefaultContributorRole() == null) {
        created.setDefaultContributorRole(ctxt.roles().findBuiltinRoleByAlias(DataverseRole.EDITOR));
    }
    // @todo for now we are saying all dataverses are permission root
    created.setPermissionRoot(true);
    if (ctxt.dataverses().findByAlias(created.getAlias()) != null) {
        throw new IllegalCommandException("A dataverse with alias " + created.getAlias() + " already exists", this);
    }
    // Save the dataverse
    Dataverse managedDv = ctxt.dataverses().save(created);
    // Find the built in admin role (currently by alias)
    DataverseRole adminRole = ctxt.roles().findBuiltinRoleByAlias(DataverseRole.ADMIN);
    String privateUrlToken = null;
    ctxt.roles().save(new RoleAssignment(adminRole, getRequest().getUser(), managedDv, privateUrlToken));
    managedDv.setPermissionModificationTime(new Timestamp(new Date().getTime()));
    managedDv = ctxt.dataverses().save(managedDv);
    ctxt.index().indexDataverse(managedDv);
    if (facetList != null) {
        ctxt.facets().deleteFacetsFor(managedDv);
        int i = 0;
        for (DatasetFieldType df : facetList) {
            ctxt.facets().create(i++, df, managedDv);
        }
    }
    if (inputLevelList != null) {
        ctxt.fieldTypeInputLevels().deleteFacetsFor(managedDv);
        for (DataverseFieldTypeInputLevel obj : inputLevelList) {
            obj.setDataverse(managedDv);
            ctxt.fieldTypeInputLevels().create(obj);
        }
    }
    return managedDv;
}

Example 20 with DatasetFieldType

use of edu.harvard.iq.dataverse.DatasetFieldType in project dataverse by IQSS.

the class IngestServiceBean method processDatasetMetadata.

private void processDatasetMetadata(FileMetadataIngest fileMetadataIngest, DatasetVersion editVersion) throws IOException {
    for (MetadataBlock mdb : editVersion.getDataset().getOwner().getMetadataBlocks()) {
        if (mdb.getName().equals(fileMetadataIngest.getMetadataBlockName())) {
            logger.fine("Ingest Service: dataset version has " + mdb.getName() + " metadata block enabled.");
            editVersion.setDatasetFields(editVersion.initDatasetFields());
            Map<String, Set<String>> fileMetadataMap = fileMetadataIngest.getMetadataMap();
            for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) {
                if (dsft.isPrimitive()) {
                    if (!dsft.isHasParent()) {
                        String dsfName = dsft.getName();
                        // See if the plugin has found anything for this field:
                        if (fileMetadataMap.get(dsfName) != null && !fileMetadataMap.get(dsfName).isEmpty()) {
                            logger.fine("Ingest Service: found extracted metadata for field " + dsfName);
                            // go through the existing fields:
                            for (DatasetField dsf : editVersion.getFlatDatasetFields()) {
                                if (dsf.getDatasetFieldType().equals(dsft)) {
                                    // yep, this is our field!
                                    // let's go through the values that the ingest
                                    // plugin found in the file for this field:
                                    Set<String> mValues = fileMetadataMap.get(dsfName);
                                    // programmatically defined. -- L.A. 4.0
                                    if (dsfName.equals("resolution.Temporal") || dsfName.equals("resolution.Spatial") || dsfName.equals("resolution.Spectral")) {
                                        // For these values, we aggregate the minimum-maximum
                                        // pair, for the entire set.
                                        // So first, we need to go through the values found by
                                        // the plugin and select the min. and max. values of
                                        // these:
                                        // (note that we are assuming that they all must
                                        // validate as doubles!)
                                        Double minValue = null;
                                        Double maxValue = null;
                                        for (String fValue : mValues) {
                                            try {
                                                double thisValue = Double.parseDouble(fValue);
                                                if (minValue == null || Double.compare(thisValue, minValue) < 0) {
                                                    minValue = thisValue;
                                                }
                                                if (maxValue == null || Double.compare(thisValue, maxValue) > 0) {
                                                    maxValue = thisValue;
                                                }
                                            } catch (NumberFormatException e) {
                                            }
                                        }
                                        // logger.fine("Min value: "+minValue+", Max value: "+maxValue);
                                        if (minValue != null && maxValue != null) {
                                            Double storedMinValue = null;
                                            Double storedMaxValue = null;
                                            String storedValue = "";
                                            if (dsf.getDatasetFieldValues() != null && dsf.getDatasetFieldValues().get(0) != null) {
                                                storedValue = dsf.getDatasetFieldValues().get(0).getValue();
                                                if (storedValue != null && !storedValue.equals("")) {
                                                    try {
                                                        if (storedValue.indexOf(" - ") > -1) {
                                                            storedMinValue = Double.parseDouble(storedValue.substring(0, storedValue.indexOf(" - ")));
                                                            storedMaxValue = Double.parseDouble(storedValue.substring(storedValue.indexOf(" - ") + 3));
                                                        } else {
                                                            storedMinValue = Double.parseDouble(storedValue);
                                                            storedMaxValue = storedMinValue;
                                                        }
                                                        if (storedMinValue != null && storedMinValue.compareTo(minValue) < 0) {
                                                            minValue = storedMinValue;
                                                        }
                                                        if (storedMaxValue != null && storedMaxValue.compareTo(maxValue) > 0) {
                                                            maxValue = storedMaxValue;
                                                        }
                                                    } catch (NumberFormatException e) {
                                                    }
                                                } else {
                                                    storedValue = "";
                                                }
                                            }
                                            // logger.fine("Stored min value: "+storedMinValue+", Stored max value: "+storedMaxValue);
                                            String newAggregateValue = "";
                                            if (minValue.equals(maxValue)) {
                                                newAggregateValue = minValue.toString();
                                            } else {
                                                newAggregateValue = minValue.toString() + " - " + maxValue.toString();
                                            }
                                            // finally, compare it to the value we have now:
                                            if (!storedValue.equals(newAggregateValue)) {
                                                if (dsf.getDatasetFieldValues() == null) {
                                                    dsf.setDatasetFieldValues(new ArrayList<DatasetFieldValue>());
                                                }
                                                if (dsf.getDatasetFieldValues().get(0) == null) {
                                                    DatasetFieldValue newDsfv = new DatasetFieldValue(dsf);
                                                    dsf.getDatasetFieldValues().add(newDsfv);
                                                }
                                                dsf.getDatasetFieldValues().get(0).setValue(newAggregateValue);
                                            }
                                        }
                                    // Ouch.
                                    } else {
                                        for (String fValue : mValues) {
                                            if (!dsft.isControlledVocabulary()) {
                                                Iterator<DatasetFieldValue> dsfvIt = dsf.getDatasetFieldValues().iterator();
                                                boolean valueExists = false;
                                                while (dsfvIt.hasNext()) {
                                                    DatasetFieldValue dsfv = dsfvIt.next();
                                                    if (fValue.equals(dsfv.getValue())) {
                                                        logger.fine("Value " + fValue + " already exists for field " + dsfName);
                                                        valueExists = true;
                                                        break;
                                                    }
                                                }
                                                if (!valueExists) {
                                                    logger.fine("Creating a new value for field " + dsfName + ": " + fValue);
                                                    DatasetFieldValue newDsfv = new DatasetFieldValue(dsf);
                                                    newDsfv.setValue(fValue);
                                                    dsf.getDatasetFieldValues().add(newDsfv);
                                                }
                                            } else {
                                                // A controlled vocabulary entry:
                                                // first, let's see if it's a legit control vocab. entry:
                                                ControlledVocabularyValue legitControlledVocabularyValue = null;
                                                Collection<ControlledVocabularyValue> definedVocabularyValues = dsft.getControlledVocabularyValues();
                                                if (definedVocabularyValues != null) {
                                                    for (ControlledVocabularyValue definedVocabValue : definedVocabularyValues) {
                                                        if (fValue.equals(definedVocabValue.getStrValue())) {
                                                            logger.fine("Yes, " + fValue + " is a valid controlled vocabulary value for the field " + dsfName);
                                                            legitControlledVocabularyValue = definedVocabValue;
                                                            break;
                                                        }
                                                    }
                                                }
                                                if (legitControlledVocabularyValue != null) {
                                                    // Only need to add the value if it is new,
                                                    // i.e. if it does not exist yet:
                                                    boolean valueExists = false;
                                                    List<ControlledVocabularyValue> existingControlledVocabValues = dsf.getControlledVocabularyValues();
                                                    if (existingControlledVocabValues != null) {
                                                        Iterator<ControlledVocabularyValue> cvvIt = existingControlledVocabValues.iterator();
                                                        while (cvvIt.hasNext()) {
                                                            ControlledVocabularyValue cvv = cvvIt.next();
                                                            if (fValue.equals(cvv.getStrValue())) {
                                                                // or should I use if (legitControlledVocabularyValue.equals(cvv)) ?
                                                                logger.fine("Controlled vocab. value " + fValue + " already exists for field " + dsfName);
                                                                valueExists = true;
                                                                break;
                                                            }
                                                        }
                                                    }
                                                    if (!valueExists) {
                                                        logger.fine("Adding controlled vocabulary value " + fValue + " to field " + dsfName);
                                                        dsf.getControlledVocabularyValues().add(legitControlledVocabularyValue);
                                                    }
                                                }
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                } else {
                    // A compound field:
                    // See if the plugin has found anything for the fields that
                    // make up this compound field; if we find at least one
                    // of the child values in the map of extracted values, we'll
                    // create a new compound field value and its child
                    // 
                    DatasetFieldCompoundValue compoundDsfv = new DatasetFieldCompoundValue();
                    int nonEmptyFields = 0;
                    for (DatasetFieldType cdsft : dsft.getChildDatasetFieldTypes()) {
                        String dsfName = cdsft.getName();
                        if (fileMetadataMap.get(dsfName) != null && !fileMetadataMap.get(dsfName).isEmpty()) {
                            logger.fine("Ingest Service: found extracted metadata for field " + dsfName + ", part of the compound field " + dsft.getName());
                            if (cdsft.isPrimitive()) {
                                // but maybe it'll change in the future.
                                if (!cdsft.isControlledVocabulary()) {
                                    // TODO: can we have controlled vocabulary
                                    // sub-fields inside compound fields?
                                    DatasetField childDsf = new DatasetField();
                                    childDsf.setDatasetFieldType(cdsft);
                                    DatasetFieldValue newDsfv = new DatasetFieldValue(childDsf);
                                    newDsfv.setValue((String) fileMetadataMap.get(dsfName).toArray()[0]);
                                    childDsf.getDatasetFieldValues().add(newDsfv);
                                    childDsf.setParentDatasetFieldCompoundValue(compoundDsfv);
                                    compoundDsfv.getChildDatasetFields().add(childDsf);
                                    nonEmptyFields++;
                                }
                            }
                        }
                    }
                    if (nonEmptyFields > 0) {
                        // actual parent for this sub-field:
                        for (DatasetField dsf : editVersion.getFlatDatasetFields()) {
                            if (dsf.getDatasetFieldType().equals(dsft)) {
                                // Now let's check that the dataset version doesn't already have
                                // this compound value - we are only interested in aggregating
                                // unique values. Note that we need to compare compound values
                                // as sets! -- i.e. all the sub fields in 2 compound fields
                                // must match in order for these 2 compounds to be recognized
                                // as "the same":
                                boolean alreadyExists = false;
                                for (DatasetFieldCompoundValue dsfcv : dsf.getDatasetFieldCompoundValues()) {
                                    int matches = 0;
                                    for (DatasetField cdsf : dsfcv.getChildDatasetFields()) {
                                        String cdsfName = cdsf.getDatasetFieldType().getName();
                                        String cdsfValue = cdsf.getDatasetFieldValues().get(0).getValue();
                                        if (cdsfValue != null && !cdsfValue.equals("")) {
                                            String extractedValue = (String) fileMetadataMap.get(cdsfName).toArray()[0];
                                            logger.fine("values: existing: " + cdsfValue + ", extracted: " + extractedValue);
                                            if (cdsfValue.equals(extractedValue)) {
                                                matches++;
                                            }
                                        }
                                    }
                                    if (matches == nonEmptyFields) {
                                        alreadyExists = true;
                                        break;
                                    }
                                }
                                if (!alreadyExists) {
                                    // save this compound value, by attaching it to the
                                    // version for proper cascading:
                                    compoundDsfv.setParentDatasetField(dsf);
                                    dsf.getDatasetFieldCompoundValues().add(compoundDsfv);
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}