Examples with DatasetFieldCompoundValue edu.harvard.iq.dataverse.DatasetFieldCompoundValue used on opensource projects

Example 6 with DatasetFieldCompoundValue

use of edu.harvard.iq.dataverse.DatasetFieldCompoundValue in project dataverse by IQSS.

the class IngestServiceBean method processDatasetMetadata.

private void processDatasetMetadata(FileMetadataIngest fileMetadataIngest, DatasetVersion editVersion) throws IOException {
    for (MetadataBlock mdb : editVersion.getDataset().getOwner().getMetadataBlocks()) {
        if (mdb.getName().equals(fileMetadataIngest.getMetadataBlockName())) {
            logger.fine("Ingest Service: dataset version has " + mdb.getName() + " metadata block enabled.");
            editVersion.setDatasetFields(editVersion.initDatasetFields());
            Map<String, Set<String>> fileMetadataMap = fileMetadataIngest.getMetadataMap();
            for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) {
                if (dsft.isPrimitive()) {
                    if (!dsft.isHasParent()) {
                        String dsfName = dsft.getName();
                        // See if the plugin has found anything for this field:
                        if (fileMetadataMap.get(dsfName) != null && !fileMetadataMap.get(dsfName).isEmpty()) {
                            logger.fine("Ingest Service: found extracted metadata for field " + dsfName);
                            // go through the existing fields:
                            for (DatasetField dsf : editVersion.getFlatDatasetFields()) {
                                if (dsf.getDatasetFieldType().equals(dsft)) {
                                    // yep, this is our field!
                                    // let's go through the values that the ingest
                                    // plugin found in the file for this field:
                                    Set<String> mValues = fileMetadataMap.get(dsfName);
                                    // programmatically defined. -- L.A. 4.0
                                    if (dsfName.equals("resolution.Temporal") || dsfName.equals("resolution.Spatial") || dsfName.equals("resolution.Spectral")) {
                                        // For these values, we aggregate the minimum-maximum
                                        // pair, for the entire set.
                                        // So first, we need to go through the values found by
                                        // the plugin and select the min. and max. values of
                                        // these:
                                        // (note that we are assuming that they all must
                                        // validate as doubles!)
                                        Double minValue = null;
                                        Double maxValue = null;
                                        for (String fValue : mValues) {
                                            try {
                                                double thisValue = Double.parseDouble(fValue);
                                                if (minValue == null || Double.compare(thisValue, minValue) < 0) {
                                                    minValue = thisValue;
                                                }
                                                if (maxValue == null || Double.compare(thisValue, maxValue) > 0) {
                                                    maxValue = thisValue;
                                                }
                                            } catch (NumberFormatException e) {
                                            }
                                        }
                                        // logger.fine("Min value: "+minValue+", Max value: "+maxValue);
                                        if (minValue != null && maxValue != null) {
                                            Double storedMinValue = null;
                                            Double storedMaxValue = null;
                                            String storedValue = "";
                                            if (dsf.getDatasetFieldValues() != null && dsf.getDatasetFieldValues().get(0) != null) {
                                                storedValue = dsf.getDatasetFieldValues().get(0).getValue();
                                                if (storedValue != null && !storedValue.equals("")) {
                                                    try {
                                                        if (storedValue.indexOf(" - ") > -1) {
                                                            storedMinValue = Double.parseDouble(storedValue.substring(0, storedValue.indexOf(" - ")));
                                                            storedMaxValue = Double.parseDouble(storedValue.substring(storedValue.indexOf(" - ") + 3));
                                                        } else {
                                                            storedMinValue = Double.parseDouble(storedValue);
                                                            storedMaxValue = storedMinValue;
                                                        }
                                                        if (storedMinValue != null && storedMinValue.compareTo(minValue) < 0) {
                                                            minValue = storedMinValue;
                                                        }
                                                        if (storedMaxValue != null && storedMaxValue.compareTo(maxValue) > 0) {
                                                            maxValue = storedMaxValue;
                                                        }
                                                    } catch (NumberFormatException e) {
                                                    }
                                                } else {
                                                    storedValue = "";
                                                }
                                            }
                                            // logger.fine("Stored min value: "+storedMinValue+", Stored max value: "+storedMaxValue);
                                            String newAggregateValue = "";
                                            if (minValue.equals(maxValue)) {
                                                newAggregateValue = minValue.toString();
                                            } else {
                                                newAggregateValue = minValue.toString() + " - " + maxValue.toString();
                                            }
                                            // finally, compare it to the value we have now:
                                            if (!storedValue.equals(newAggregateValue)) {
                                                if (dsf.getDatasetFieldValues() == null) {
                                                    dsf.setDatasetFieldValues(new ArrayList<DatasetFieldValue>());
                                                }
                                                if (dsf.getDatasetFieldValues().get(0) == null) {
                                                    DatasetFieldValue newDsfv = new DatasetFieldValue(dsf);
                                                    dsf.getDatasetFieldValues().add(newDsfv);
                                                }
                                                dsf.getDatasetFieldValues().get(0).setValue(newAggregateValue);
                                            }
                                        }
                                    // Ouch.
                                    } else {
                                        for (String fValue : mValues) {
                                            if (!dsft.isControlledVocabulary()) {
                                                Iterator<DatasetFieldValue> dsfvIt = dsf.getDatasetFieldValues().iterator();
                                                boolean valueExists = false;
                                                while (dsfvIt.hasNext()) {
                                                    DatasetFieldValue dsfv = dsfvIt.next();
                                                    if (fValue.equals(dsfv.getValue())) {
                                                        logger.fine("Value " + fValue + " already exists for field " + dsfName);
                                                        valueExists = true;
                                                        break;
                                                    }
                                                }
                                                if (!valueExists) {
                                                    logger.fine("Creating a new value for field " + dsfName + ": " + fValue);
                                                    DatasetFieldValue newDsfv = new DatasetFieldValue(dsf);
                                                    newDsfv.setValue(fValue);
                                                    dsf.getDatasetFieldValues().add(newDsfv);
                                                }
                                            } else {
                                                // A controlled vocabulary entry:
                                                // first, let's see if it's a legit control vocab. entry:
                                                ControlledVocabularyValue legitControlledVocabularyValue = null;
                                                Collection<ControlledVocabularyValue> definedVocabularyValues = dsft.getControlledVocabularyValues();
                                                if (definedVocabularyValues != null) {
                                                    for (ControlledVocabularyValue definedVocabValue : definedVocabularyValues) {
                                                        if (fValue.equals(definedVocabValue.getStrValue())) {
                                                            logger.fine("Yes, " + fValue + " is a valid controlled vocabulary value for the field " + dsfName);
                                                            legitControlledVocabularyValue = definedVocabValue;
                                                            break;
                                                        }
                                                    }
                                                }
                                                if (legitControlledVocabularyValue != null) {
                                                    // Only need to add the value if it is new,
                                                    // i.e. if it does not exist yet:
                                                    boolean valueExists = false;
                                                    List<ControlledVocabularyValue> existingControlledVocabValues = dsf.getControlledVocabularyValues();
                                                    if (existingControlledVocabValues != null) {
                                                        Iterator<ControlledVocabularyValue> cvvIt = existingControlledVocabValues.iterator();
                                                        while (cvvIt.hasNext()) {
                                                            ControlledVocabularyValue cvv = cvvIt.next();
                                                            if (fValue.equals(cvv.getStrValue())) {
                                                                // or should I use if (legitControlledVocabularyValue.equals(cvv)) ?
                                                                logger.fine("Controlled vocab. value " + fValue + " already exists for field " + dsfName);
                                                                valueExists = true;
                                                                break;
                                                            }
                                                        }
                                                    }
                                                    if (!valueExists) {
                                                        logger.fine("Adding controlled vocabulary value " + fValue + " to field " + dsfName);
                                                        dsf.getControlledVocabularyValues().add(legitControlledVocabularyValue);
                                                    }
                                                }
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                } else {
                    // A compound field:
                    // See if the plugin has found anything for the fields that
                    // make up this compound field; if we find at least one
                    // of the child values in the map of extracted values, we'll
                    // create a new compound field value and its child
                    // 
                    DatasetFieldCompoundValue compoundDsfv = new DatasetFieldCompoundValue();
                    int nonEmptyFields = 0;
                    for (DatasetFieldType cdsft : dsft.getChildDatasetFieldTypes()) {
                        String dsfName = cdsft.getName();
                        if (fileMetadataMap.get(dsfName) != null && !fileMetadataMap.get(dsfName).isEmpty()) {
                            logger.fine("Ingest Service: found extracted metadata for field " + dsfName + ", part of the compound field " + dsft.getName());
                            if (cdsft.isPrimitive()) {
                                // but maybe it'll change in the future.
                                if (!cdsft.isControlledVocabulary()) {
                                    // TODO: can we have controlled vocabulary
                                    // sub-fields inside compound fields?
                                    DatasetField childDsf = new DatasetField();
                                    childDsf.setDatasetFieldType(cdsft);
                                    DatasetFieldValue newDsfv = new DatasetFieldValue(childDsf);
                                    newDsfv.setValue((String) fileMetadataMap.get(dsfName).toArray()[0]);
                                    childDsf.getDatasetFieldValues().add(newDsfv);
                                    childDsf.setParentDatasetFieldCompoundValue(compoundDsfv);
                                    compoundDsfv.getChildDatasetFields().add(childDsf);
                                    nonEmptyFields++;
                                }
                            }
                        }
                    }
                    if (nonEmptyFields > 0) {
                        // actual parent for this sub-field:
                        for (DatasetField dsf : editVersion.getFlatDatasetFields()) {
                            if (dsf.getDatasetFieldType().equals(dsft)) {
                                // Now let's check that the dataset version doesn't already have
                                // this compound value - we are only interested in aggregating
                                // unique values. Note that we need to compare compound values
                                // as sets! -- i.e. all the sub fields in 2 compound fields
                                // must match in order for these 2 compounds to be recognized
                                // as "the same":
                                boolean alreadyExists = false;
                                for (DatasetFieldCompoundValue dsfcv : dsf.getDatasetFieldCompoundValues()) {
                                    int matches = 0;
                                    for (DatasetField cdsf : dsfcv.getChildDatasetFields()) {
                                        String cdsfName = cdsf.getDatasetFieldType().getName();
                                        String cdsfValue = cdsf.getDatasetFieldValues().get(0).getValue();
                                        if (cdsfValue != null && !cdsfValue.equals("")) {
                                            String extractedValue = (String) fileMetadataMap.get(cdsfName).toArray()[0];
                                            logger.fine("values: existing: " + cdsfValue + ", extracted: " + extractedValue);
                                            if (cdsfValue.equals(extractedValue)) {
                                                matches++;
                                            }
                                        }
                                    }
                                    if (matches == nonEmptyFields) {
                                        alreadyExists = true;
                                        break;
                                    }
                                }
                                if (!alreadyExists) {
                                    // save this compound value, by attaching it to the
                                    // version for proper cascading:
                                    compoundDsfv.setParentDatasetField(dsf);
                                    dsf.getDatasetFieldCompoundValues().add(compoundDsfv);
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}

Example 7 with DatasetFieldCompoundValue

use of edu.harvard.iq.dataverse.DatasetFieldCompoundValue in project dataverse by IQSS.

the class JsonPrinterTest method testDatasetContactWithPrivacy.

@Test
public void testDatasetContactWithPrivacy() {
    MetadataBlock block = new MetadataBlock();
    block.setName("citation");
    List<DatasetField> fields = new ArrayList<>();
    DatasetField datasetContactField = new DatasetField();
    DatasetFieldType datasetContactDatasetFieldType = datasetFieldTypeSvc.findByName("datasetContact");
    datasetContactDatasetFieldType.setMetadataBlock(block);
    datasetContactField.setDatasetFieldType(datasetContactDatasetFieldType);
    List<DatasetFieldCompoundValue> vals = new LinkedList<>();
    DatasetFieldCompoundValue val = new DatasetFieldCompoundValue();
    val.setParentDatasetField(datasetContactField);
    val.setChildDatasetFields(Arrays.asList(constructPrimitive("datasetContactEmail", "foo@bar.com"), constructPrimitive("datasetContactName", "Foo Bar"), constructPrimitive("datasetContactAffiliation", "Bar University")));
    vals.add(val);
    datasetContactField.setDatasetFieldCompoundValues(vals);
    fields.add(datasetContactField);
    JsonPrinter jsonPrinter = new JsonPrinter(new MockSettingsSvc());
    JsonObject jsonObject = JsonPrinter.json(block, fields).build();
    assertNotNull(jsonObject);
    System.out.println("json: " + JsonUtil.prettyPrint(jsonObject.toString()));
    assertEquals("Foo Bar", jsonObject.getJsonArray("fields").getJsonObject(0).getJsonArray("value").getJsonObject(0).getJsonObject("datasetContactName").getString("value"));
    assertEquals("Bar University", jsonObject.getJsonArray("fields").getJsonObject(0).getJsonArray("value").getJsonObject(0).getJsonObject("datasetContactAffiliation").getString("value"));
    assertEquals(null, jsonObject.getJsonArray("fields").getJsonObject(0).getJsonArray("value").getJsonObject(0).getJsonObject("datasetContactEmail"));
    JsonObject byBlocks = jsonPrinter.jsonByBlocks(fields).build();
    System.out.println("byBlocks: " + JsonUtil.prettyPrint(byBlocks.toString()));
    assertEquals("Foo Bar", byBlocks.getJsonObject("citation").getJsonArray("fields").getJsonObject(0).getJsonArray("value").getJsonObject(0).getJsonObject("datasetContactName").getString("value"));
    assertEquals("Bar University", byBlocks.getJsonObject("citation").getJsonArray("fields").getJsonObject(0).getJsonArray("value").getJsonObject(0).getJsonObject("datasetContactAffiliation").getString("value"));
    assertEquals(null, byBlocks.getJsonObject("citation").getJsonArray("fields").getJsonObject(0).getJsonArray("value").getJsonObject(0).getJsonObject("datasetContactEmail"));
}

Example 8 with DatasetFieldCompoundValue

use of edu.harvard.iq.dataverse.DatasetFieldCompoundValue in project dataverse by IQSS.

the class ForeignMetadataImportServiceBean method processXMLElement.

private void processXMLElement(XMLStreamReader xmlr, String currentPath, String openingTag, ForeignMetadataFormatMapping foreignFormatMapping, DatasetVersion datasetVersion) throws XMLStreamException {
    logger.fine("entering processXMLElement; (" + currentPath + ")");
    for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) {
        if (event == XMLStreamConstants.START_ELEMENT) {
            String currentElement = xmlr.getLocalName();
            ForeignMetadataFieldMapping mappingDefined = datasetfieldService.findFieldMapping(foreignFormatMapping.getName(), currentPath + currentElement);
            if (mappingDefined != null) {
                DatasetFieldCompoundValue cachedCompoundValue = null;
                for (ForeignMetadataFieldMapping childMapping : mappingDefined.getChildFieldMappings()) {
                    if (childMapping.isAttribute()) {
                        String attributeName = childMapping.getForeignFieldXPath();
                        String attributeValue = xmlr.getAttributeValue(null, attributeName);
                        if (attributeValue != null) {
                            String mappedFieldName = childMapping.getDatasetfieldName();
                            logger.fine("looking up dataset field " + mappedFieldName);
                            DatasetFieldType mappedFieldType = datasetfieldService.findByNameOpt(mappedFieldName);
                            if (mappedFieldType != null) {
                                try {
                                    cachedCompoundValue = createDatasetFieldValue(mappedFieldType, cachedCompoundValue, attributeValue, datasetVersion);
                                } catch (Exception ex) {
                                    logger.warning("Caught unknown exception when processing attribute " + currentPath + currentElement + "{" + attributeName + "} (skipping);");
                                }
                            } else {
                                throw new EJBException("Bad foreign metadata field mapping: no such DatasetField " + mappedFieldName + "!");
                            }
                        }
                    }
                }
                // Process the payload of this XML element:
                String dataverseFieldName = mappingDefined.getDatasetfieldName();
                if (dataverseFieldName != null && !dataverseFieldName.equals("")) {
                    DatasetFieldType dataverseFieldType = datasetfieldService.findByNameOpt(dataverseFieldName);
                    if (dataverseFieldType != null) {
                        String elementTextPayload = parseText(xmlr);
                        createDatasetFieldValue(dataverseFieldType, cachedCompoundValue, elementTextPayload, datasetVersion);
                    } else {
                        throw new EJBException("Bad foreign metadata field mapping: no such DatasetField " + dataverseFieldName + "!");
                    }
                }
            } else {
                // recursively, process the xml stream further down:
                processXMLElement(xmlr, currentPath + currentElement + ":", currentElement, foreignFormatMapping, datasetVersion);
            }
        } else if (event == XMLStreamConstants.END_ELEMENT) {
            if (xmlr.getLocalName().equals(openingTag))
                return;
        }
    }
}

Example 9 with DatasetFieldCompoundValue

use of edu.harvard.iq.dataverse.DatasetFieldCompoundValue in project dataverse by IQSS.

the class ForeignMetadataImportServiceBean method createDatasetFieldValue.

private DatasetFieldCompoundValue createDatasetFieldValue(DatasetFieldType dsft, DatasetFieldCompoundValue savedCompoundValue, String elementText, DatasetVersion datasetVersion) {
    if (dsft.isPrimitive()) {
        if (!dsft.isHasParent()) {
            // simple primitive:
            DatasetField dsf = null;
            for (DatasetField existingDsf : datasetVersion.getFlatDatasetFields()) {
                if (existingDsf.getDatasetFieldType().equals(dsft)) {
                    dsf = existingDsf;
                }
            }
            // if doesn't exist, create a new one:
            if (dsf == null) {
                dsf = new DatasetField();
                dsf.setDatasetFieldType(dsft);
                datasetVersion.getDatasetFields().add(dsf);
                dsf.setDatasetVersion(datasetVersion);
            }
            String dsfName = dsft.getName();
            if (!dsft.isControlledVocabulary()) {
                logger.fine("Creating a new value for field " + dsfName + ": " + elementText);
                DatasetFieldValue newDsfv = new DatasetFieldValue(dsf);
                newDsfv.setValue(elementText);
                dsf.getDatasetFieldValues().add(newDsfv);
            } else {
            // A controlled vocabulary entry:
            // first, let's see if it's a legit control vocab. entry:
            /* not supported yet; though I expect the commented-out code
                       below to work;
                    ControlledVocabularyValue legitControlledVocabularyValue = null;
                    Collection<ControlledVocabularyValue> definedVocabularyValues = dsft.getControlledVocabularyValues();
                    if (definedVocabularyValues != null) {
                        for (ControlledVocabularyValue definedVocabValue : definedVocabularyValues) {
                            if (elementText.equals(definedVocabValue.getStrValue())) {
                                logger.fine("Yes, " + elementText + " is a valid controlled vocabulary value for the field " + dsfName);
                                legitControlledVocabularyValue = definedVocabValue;
                                break;
                            }
                        }
                    }
                    if (legitControlledVocabularyValue != null) {
                        logger.fine("Adding controlled vocabulary value " + elementText + " to field " + dsfName);
                        dsf.getControlledVocabularyValues().add(legitControlledVocabularyValue);
                    }
                    */
            }
            // No compound values had to be created; returning null:
            return null;
        } else {
            // a primitive that is part of a compound value:
            // first, let's create the field and the value, for the
            // primitive node itself:
            DatasetField childField = new DatasetField();
            childField.setDatasetFieldType(dsft);
            DatasetFieldValue childValue = new DatasetFieldValue(childField);
            childValue.setValue(elementText);
            childField.getDatasetFieldValues().add(childValue);
            // see if a compound value of the right type has already been
            // created and passed to us:
            DatasetFieldCompoundValue parentCompoundValue = null;
            DatasetFieldType parentFieldType = dsft.getParentDatasetFieldType();
            if (parentFieldType == null) {
                logger.severe("Child field type with no parent field type defined!");
                // could just skip this field and try to continue - ?
                return null;
            }
            if (savedCompoundValue != null) {
                if (parentFieldType.equals(savedCompoundValue.getParentDatasetField().getDatasetFieldType())) {
                    parentCompoundValue = savedCompoundValue;
                }
            }
            if (parentCompoundValue == null) {
                // and to do that, we need to find or create the "parent"
                // dataset field for this compoound value:
                // (I put quotes around "parent", because I really feel it
                // is a misnomer, and that the relationship between the compound value
                // and the corresponding dataset field should be called
                // "CompoundDatasetField", not "ParentDatasetField") (discuss?)
                DatasetField parentField = null;
                for (DatasetField existingDsf : datasetVersion.getFlatDatasetFields()) {
                    if (existingDsf.getDatasetFieldType().equals(parentFieldType)) {
                        parentField = existingDsf;
                    }
                }
                // if doesn't exist, create a new one:
                if (parentField == null) {
                    parentField = new DatasetField();
                    parentField.setDatasetFieldType(parentFieldType);
                    datasetVersion.getDatasetFields().add(parentField);
                    parentField.setDatasetVersion(datasetVersion);
                }
                // and then create new compound value:
                parentCompoundValue = new DatasetFieldCompoundValue();
                parentCompoundValue.setParentDatasetField(parentField);
                parentField.getDatasetFieldCompoundValues().add(parentCompoundValue);
            }
            childField.setParentDatasetFieldCompoundValue(parentCompoundValue);
            parentCompoundValue.getChildDatasetFields().add(childField);
            return parentCompoundValue;
        }
    }
    return null;
}

Example 10 with DatasetFieldCompoundValue

use of edu.harvard.iq.dataverse.DatasetFieldCompoundValue in project dataverse by IQSS.

the class JsonParserTest method testCompoundRepeatsRoundtrip.

@Test
public void testCompoundRepeatsRoundtrip() throws JsonParseException {
    DatasetField expected = new DatasetField();
    expected.setDatasetFieldType(datasetFieldTypeSvc.findByName("coordinate"));
    List<DatasetFieldCompoundValue> vals = new LinkedList<>();
    for (int i = 0; i < 5; i++) {
        DatasetFieldCompoundValue val = new DatasetFieldCompoundValue();
        val.setParentDatasetField(expected);
        val.setChildDatasetFields(Arrays.asList(latLonField("lat", Integer.toString(i * 10)), latLonField("lon", Integer.toString(3 + i * 10))));
        vals.add(val);
    }
    expected.setDatasetFieldCompoundValues(vals);
    JsonObject json = JsonPrinter.json(expected);
    System.out.println("json = " + json);
    DatasetField actual = sut.parseField(json);
    assertFieldsEqual(expected, actual);
}