Examples with SummaryStatistic edu.harvard.iq.dataverse.datavariable.SummaryStatistic used on opensource projects

Example 1 with SummaryStatistic

use of edu.harvard.iq.dataverse.datavariable.SummaryStatistic in project dataverse by IQSS.

the class DTA117FileReader method readVariableTypes.

/* 
     * Variable type information is stored in the <variable_types>...</variable_types>
     * section, as number_of_variables * 2 byte values. 
     * the type codes are defined as follows: 
     * (TODO: ...)
    */
private void readVariableTypes(DataReader reader) throws IOException {
    // TODO:
    // check that we are at the right byte offset!
    logger.fine("Type section; at offset " + reader.getByteOffset() + "; dta map offset: " + dtaMap.getOffset_types());
    reader.readOpeningTag(TAG_VARIABLE_TYPES);
    List<DataVariable> variableList = new ArrayList<DataVariable>();
    // setup variableTypeList
    variableTypes = new String[dataTable.getVarQuantity().intValue()];
    for (int i = 0; i < dataTable.getVarQuantity(); i++) {
        int type = reader.readShortInteger();
        logger.fine("variable " + i + ": type=" + type);
        DataVariable dv = new DataVariable();
        dv.setInvalidRanges(new ArrayList<VariableRange>());
        dv.setSummaryStatistics(new ArrayList<SummaryStatistic>());
        dv.setCategories(new ArrayList<VariableCategory>());
        dv.setUnf("UNF:pending");
        dv.setFileOrder(i);
        dv.setDataTable(dataTable);
        variableTypes[i] = configureVariableType(dv, type);
        // TODO:
        // we could also calculate the byte offset table now, rather
        // then figure it out later... - ?
        variableList.add(dv);
    }
    reader.readClosingTag(TAG_VARIABLE_TYPES);
    dataTable.setDataVariables(variableList);
}

Example 2 with SummaryStatistic

use of edu.harvard.iq.dataverse.datavariable.SummaryStatistic in project dataverse by IQSS.

the class SAVFileReader method decodeRecordType2.

void decodeRecordType2(BufferedInputStream stream) throws IOException {
    dbgLog.fine("decodeRecordType2(): start");
    if (stream == null) {
        throw new IllegalArgumentException("stream == null!");
    }
    Map<String, String> printFormatNameTable = new LinkedHashMap<String, String>();
    Map<String, String> variableLabelMap = new LinkedHashMap<String, String>();
    Map<String, List<String>> missingValueTable = new LinkedHashMap<String, List<String>>();
    List<Integer> printFormatList = new ArrayList<Integer>();
    String caseWeightVariableName = null;
    int caseWeightVariableIndex = 0;
    boolean lastVariableIsExtendable = false;
    boolean extendedVariableMode = false;
    boolean obs255 = false;
    String lastVariableName = null;
    String lastExtendedVariable = null;
    // this field repeats as many as the number of variables in
    // this sav file
    // (note that the above statement is not technically correct, this
    // record repeats not just for every variable in the file, but for
    // every OBS (8 byte unit); i.e., if a string is split into multiple
    // OBS units, each one will have its own RT2 record -- L.A.).
    // Each field constists of a fixed (32-byte) segment and
    // then a few variable segments:
    // if the variable has a label (3rd INT4 set to 1), then there's 4 more
    // bytes specifying the length of the label, and then that many bytes
    // holding the label itself (no more than 256).
    // Then if there are optional missing value units (4th INT4 set to 1)
    // there will be 3 more OBS units attached = 24 extra bytes.
    int variableCounter = 0;
    int obsSeqNumber = 0;
    int j;
    dbgLog.fine("RT2: Reading " + OBSUnitsPerCase + " OBS units.");
    for (j = 0; j < OBSUnitsPerCase; j++) {
        dbgLog.fine("RT2: " + j + "-th RT2 unit is being decoded.");
        // 2.0: read the fixed[=non-optional] 32-byte segment
        byte[] recordType2Fixed = new byte[LENGTH_RECORDTYPE2_FIXED];
        try {
            int nbytes = stream.read(recordType2Fixed, 0, LENGTH_RECORDTYPE2_FIXED);
            if (nbytes == 0) {
                throw new IOException("reading recordType2: no bytes read!");
            }
            int offset = 0;
            // 2.1: create int-view of the bytebuffer for the first 16-byte segment
            int rt2_1st_4_units = 4;
            ByteBuffer[] bb_record_type2_fixed_part1 = new ByteBuffer[rt2_1st_4_units];
            int[] recordType2FixedPart1 = new int[rt2_1st_4_units];
            for (int i = 0; i < rt2_1st_4_units; i++) {
                bb_record_type2_fixed_part1[i] = ByteBuffer.wrap(recordType2Fixed, offset, LENGTH_SAV_INT_BLOCK);
                offset += LENGTH_SAV_INT_BLOCK;
                if (isLittleEndian) {
                    bb_record_type2_fixed_part1[i].order(ByteOrder.LITTLE_ENDIAN);
                }
                recordType2FixedPart1[i] = bb_record_type2_fixed_part1[i].getInt();
            }
            // 1st ([0]) element must be 2 otherwise no longer Record Type 2
            if (recordType2FixedPart1[0] != 2) {
                dbgLog.warning(j + "-th RT header value is no longet RT2! " + recordType2FixedPart1[0]);
                break;
            }
            dbgLog.fine("variable type[must be 2]=" + recordType2FixedPart1[0]);
            // 2.3 variable name: 8 byte(space[x20]-padded)
            // This field is located at the very end of the 32 byte
            // fixed-size RT2 header (bytes 24-31).
            // We are processing it now, so that
            // we can make the decision on whether this variable is part
            // of a compound variable:
            String RawVariableName = getNullStrippedString(new String(Arrays.copyOfRange(recordType2Fixed, 24, (24 + LENGTH_VARIABLE_NAME)), defaultCharSet));
            // offset +=LENGTH_VARIABLE_NAME;
            String variableName = null;
            if (RawVariableName.indexOf(' ') >= 0) {
                variableName = RawVariableName.substring(0, RawVariableName.indexOf(' '));
            } else {
                variableName = RawVariableName;
            }
            // 2nd ([1]) element: numeric variable = 0 :for string variable
            // this block indicates its datum-length, i.e, >0 ;
            // if -1, this RT2 unit is a non-1st RT2 unit for a string variable
            // whose value is longer than 8 character.
            boolean isNumericVariable = false;
            dbgLog.fine("variable type(0: numeric; > 0: String;-1 continue )=" + recordType2FixedPart1[1]);
            // OBSwiseTypelList.add(recordType2FixedPart1[1]);
            int HowManyRt2Units = 1;
            if (recordType2FixedPart1[1] == -1) {
                dbgLog.fine("this RT2 is an 8 bit continuation chunk of an earlier string variable");
                if (obs255) {
                    if (obsSeqNumber < 30) {
                        OBSwiseTypelList.add(recordType2FixedPart1[1]);
                        obsSeqNumber++;
                    } else {
                        OBSwiseTypelList.add(-2);
                        obs255 = false;
                        obsSeqNumber = 0;
                    }
                } else {
                    OBSwiseTypelList.add(recordType2FixedPart1[1]);
                }
                obsNonVariableBlockSet.add(j);
                continue;
            } else if (recordType2FixedPart1[1] == 0) {
                // This is a numeric variable
                extendedVariableMode = false;
                // And as such, it cannot be an extension of a
                // previous, long string variable.
                OBSwiseTypelList.add(recordType2FixedPart1[1]);
                variableCounter++;
                isNumericVariable = true;
                variableTypelList.add(recordType2FixedPart1[1]);
            } else if (recordType2FixedPart1[1] > 0) {
                if (recordType2FixedPart1[1] == 255) {
                    obs255 = true;
                }
                if (lastVariableIsExtendable) {
                    String varNameBase = null;
                    if (lastVariableName.length() > 5) {
                        varNameBase = lastVariableName.substring(0, 5);
                    } else {
                        varNameBase = lastVariableName;
                    }
                    if (extendedVariableMode) {
                        if (variableNameIsAnIncrement(varNameBase, lastExtendedVariable, variableName)) {
                            OBSwiseTypelList.add(-1);
                            lastExtendedVariable = variableName;
                        // OK, we stay in the "extended variable" mode;
                        // but we can't move on to the next OBS (hence the commented out
                        // "continue" below:
                        // continue;
                        // see the next comment below for the explanation.
                        // 
                        // Should we also set "extendable" flag to false at this point
                        // if it's shorter than 255 bytes, i.e. the last extended chunk?
                        } else {
                            extendedVariableMode = false;
                        }
                    } else {
                        if (variableNameIsAnIncrement(varNameBase, variableName)) {
                            OBSwiseTypelList.add(-1);
                            extendedVariableMode = true;
                            dbgLog.fine("RT2: in extended variable mode; variable " + variableName);
                            lastExtendedVariable = variableName;
                        // Before we move on to the next OBS unit, we need to check
                        // if this current extended variable has its own label specified;
                        // If so, we need to determine its length, then read and skip
                        // that many bytes.
                        // Hence the commented out "continue" below:
                        // continue;
                        }
                    }
                }
                if (!extendedVariableMode) {
                    // OK, this is a "real"
                    // string variable, and not a continuation chunk of a compound
                    // string.
                    OBSwiseTypelList.add(recordType2FixedPart1[1]);
                    variableCounter++;
                    if (recordType2FixedPart1[1] == 255) {
                        // This variable is 255 bytes long, i.e. this is
                        // either the single "atomic" variable of the
                        // max allowed size, or it's a 255 byte segment
                        // of a compound variable. So we will check
                        // the next variable and see if it is the continuation
                        // of this one.
                        lastVariableIsExtendable = true;
                    } else {
                        lastVariableIsExtendable = false;
                    }
                    if (recordType2FixedPart1[1] % LENGTH_SAV_OBS_BLOCK == 0) {
                        HowManyRt2Units = recordType2FixedPart1[1] / LENGTH_SAV_OBS_BLOCK;
                    } else {
                        HowManyRt2Units = recordType2FixedPart1[1] / LENGTH_SAV_OBS_BLOCK + 1;
                    }
                    variableTypelList.add(recordType2FixedPart1[1]);
                }
            }
            if (!extendedVariableMode) {
                // Again, we only want to do the following steps for the "real"
                // variables, not the chunks of split mega-variables:
                dbgLog.fine("RT2: HowManyRt2Units for this variable=" + HowManyRt2Units);
                lastVariableName = variableName;
                // caseWeightVariableOBSIndex starts from 1: 0 is used for does-not-exist cases
                if (j == (caseWeightVariableOBSIndex - 1)) {
                    caseWeightVariableName = variableName;
                    // TODO: do we need this "index"? -- 4.0 alpha
                    caseWeightVariableIndex = variableCounter;
                // /smd.setCaseWeightVariableName(caseWeightVariableName);
                // /smd.getFileInformation().put("caseWeightVariableIndex", caseWeightVariableIndex);
                }
                OBSIndexToVariableName.put(j, variableName);
                // dbgLog.fine("\nvariable name="+variableName+"<-");
                dbgLog.fine("RT2: " + j + "-th variable name=" + variableName + "<-");
                dbgLog.fine("RT2: raw variable: " + RawVariableName);
                variableNameList.add(variableName);
            }
            // 3rd ([2]) element: = 1 variable-label block follows; 0 = no label
            // 
            dbgLog.fine("RT: variable label follows?(1:yes; 0: no)=" + recordType2FixedPart1[2]);
            boolean hasVariableLabel = recordType2FixedPart1[2] == 1 ? true : false;
            if ((recordType2FixedPart1[2] != 0) && (recordType2FixedPart1[2] != 1)) {
                throw new IOException("RT2: reading error: value is neither 0 or 1" + recordType2FixedPart1[2]);
            }
            if (hasVariableLabel) {
                byte[] length_variable_label = new byte[4];
                int nbytes_2_4 = stream.read(length_variable_label);
                if (nbytes_2_4 == 0) {
                    throw new IOException("RT 2: error reading recordType2.4: no bytes read!");
                } else {
                    dbgLog.fine("nbytes_2_4=" + nbytes_2_4);
                }
                ByteBuffer bb_length_variable_label = ByteBuffer.wrap(length_variable_label, 0, LENGTH_VARIABLE_LABEL);
                if (isLittleEndian) {
                    bb_length_variable_label.order(ByteOrder.LITTLE_ENDIAN);
                }
                int rawVariableLabelLength = bb_length_variable_label.getInt();
                dbgLog.fine("rawVariableLabelLength=" + rawVariableLabelLength);
                int variableLabelLength = getSAVintAdjustedBlockLength(rawVariableLabelLength);
                dbgLog.fine("RT2: variableLabelLength=" + variableLabelLength);
                // 2.5 [optional]variable label whose length is found at 2.4
                String variableLabel = "";
                if (rawVariableLabelLength > 0) {
                    byte[] variable_label = new byte[variableLabelLength];
                    int nbytes_2_5 = stream.read(variable_label);
                    if (nbytes_2_5 == 0) {
                        throw new IOException("RT 2: error reading recordType2.5: " + variableLabelLength + " bytes requested, no bytes read!");
                    } else {
                        dbgLog.fine("nbytes_2_5=" + nbytes_2_5);
                    }
                    variableLabel = getNullStrippedString(new String(Arrays.copyOfRange(variable_label, 0, rawVariableLabelLength), defaultCharSet));
                    dbgLog.fine("RT2: variableLabel=" + variableLabel + "<-");
                    dbgLog.fine(variableName + " => " + variableLabel);
                } else {
                    dbgLog.fine("RT2: defaulting to empty variable label.");
                }
                if (!extendedVariableMode) {
                    // We only have any use for this label if it's a "real" variable.
                    // Thinking about it, it doesn't make much sense for the "fake"
                    // variables that are actually chunks of large strings to store
                    // their own labels. But in some files they do. Then failing to read
                    // the bytes would result in getting out of sync with the RT record
                    // borders. So we always read the bytes, but only use them for
                    // the real variable entries.
                    /*String variableLabel = new String(Arrays.copyOfRange(variable_label,
                                0, rawVariableLabelLength),"US-ASCII");*/
                    variableLabelMap.put(variableName, variableLabel);
                }
            }
            if (extendedVariableMode) {
                // that's how SPSS stores them.
                continue;
            }
            // 4th ([3]) element: Missing value type code
            // 0[none], 1, 2, 3 [point-type],-2[range], -3 [range type+ point]
            dbgLog.fine("RT: missing value unit follows?(if 0, none)=" + recordType2FixedPart1[3]);
            boolean hasMissingValues = (validMissingValueCodeSet.contains(recordType2FixedPart1[3]) && (recordType2FixedPart1[3] != 0)) ? true : false;
            InvalidData invalidDataInfo = null;
            if (recordType2FixedPart1[3] != 0) {
                invalidDataInfo = new InvalidData(recordType2FixedPart1[3]);
                dbgLog.fine("RT: missing value type=" + invalidDataInfo.getType());
            }
            // 2.2: print/write formats: 4-byte each = 8 bytes
            byte[] printFormt = Arrays.copyOfRange(recordType2Fixed, offset, offset + LENGTH_PRINT_FORMAT_CODE);
            dbgLog.fine("printFrmt=" + new String(Hex.encodeHex(printFormt)));
            offset += LENGTH_PRINT_FORMAT_CODE;
            int formatCode = isLittleEndian ? printFormt[2] : printFormt[1];
            int formatWidth = isLittleEndian ? printFormt[1] : printFormt[2];
            // TODO:
            // What should we be doing with these "format decimal positions"
            // in 4.0?
            // -- L.A. 4.0 alpha
            int formatDecimalPointPosition = isLittleEndian ? printFormt[0] : printFormt[3];
            dbgLog.fine("RT2: format code{5=F, 1=A[String]}=" + formatCode);
            formatDecimalPointPositionList.add(formatDecimalPointPosition);
            if (!SPSSConstants.FORMAT_CODE_TABLE_SAV.containsKey(formatCode)) {
                throw new IOException("Unknown format code was found = " + formatCode);
            } else {
                printFormatList.add(formatCode);
            }
            byte[] writeFormt = Arrays.copyOfRange(recordType2Fixed, offset, offset + LENGTH_WRITE_FORMAT_CODE);
            dbgLog.fine("RT2: writeFrmt=" + new String(Hex.encodeHex(writeFormt)));
            if (writeFormt[3] != 0x00) {
                dbgLog.fine("byte-order(write format): reversal required");
            }
            offset += LENGTH_WRITE_FORMAT_CODE;
            if (!SPSSConstants.ORDINARY_FORMAT_CODE_SET.contains(formatCode)) {
                StringBuilder sb = new StringBuilder(SPSSConstants.FORMAT_CODE_TABLE_SAV.get(formatCode) + formatWidth);
                if (formatDecimalPointPosition > 0) {
                    sb.append("." + formatDecimalPointPosition);
                }
                dbgLog.fine("formattable[i] = " + variableName + " -> " + sb.toString());
                printFormatNameTable.put(variableName, sb.toString());
            }
            printFormatTable.put(variableName, SPSSConstants.FORMAT_CODE_TABLE_SAV.get(formatCode));
            if (hasMissingValues) {
                dbgLog.fine("RT2: decoding missing value: type=" + recordType2FixedPart1[3]);
                int howManyMissingValueUnits = missingValueCodeUnits.get(recordType2FixedPart1[3]);
                // int howManyMissingValueUnits = recordType2FixedPart1[3] > 0 ? recordType2FixedPart1[3] :  0;
                dbgLog.fine("RT2: howManyMissingValueUnits=" + howManyMissingValueUnits);
                byte[] missing_value_code_units = new byte[LENGTH_SAV_OBS_BLOCK * howManyMissingValueUnits];
                int nbytes_2_6 = stream.read(missing_value_code_units);
                if (nbytes_2_6 == 0) {
                    throw new IOException("RT 2: reading recordType2.6: no byte was read");
                } else {
                    dbgLog.fine("nbytes_2_6=" + nbytes_2_6);
                }
                if (isNumericVariable) {
                    double[] missingValues = new double[howManyMissingValueUnits];
                    // List<String> mvp = new ArrayList<String>();
                    List<String> mv = new ArrayList<String>();
                    ByteBuffer[] bb_missig_value_code = new ByteBuffer[howManyMissingValueUnits];
                    int offset_start = 0;
                    for (int i = 0; i < howManyMissingValueUnits; i++) {
                        bb_missig_value_code[i] = ByteBuffer.wrap(missing_value_code_units, offset_start, LENGTH_SAV_OBS_BLOCK);
                        offset_start += LENGTH_SAV_OBS_BLOCK;
                        if (isLittleEndian) {
                            bb_missig_value_code[i].order(ByteOrder.LITTLE_ENDIAN);
                        }
                        ByteBuffer temp = bb_missig_value_code[i].duplicate();
                        missingValues[i] = bb_missig_value_code[i].getDouble();
                        if (Double.toHexString(missingValues[i]).equals("-0x1.ffffffffffffep1023")) {
                            dbgLog.fine("1st value is LOWEST");
                            mv.add(Double.toHexString(missingValues[i]));
                        } else if (Double.valueOf(missingValues[i]).equals(Double.MAX_VALUE)) {
                            dbgLog.fine("2nd value is HIGHEST");
                            mv.add(Double.toHexString(missingValues[i]));
                        } else {
                            mv.add(doubleNumberFormatter.format(missingValues[i]));
                        }
                        dbgLog.fine(i + "-th missing value=" + Double.toHexString(missingValues[i]));
                    }
                    dbgLog.fine("variableName=" + variableName);
                    if (recordType2FixedPart1[3] > 0) {
                        // point cases only
                        dbgLog.fine("mv(>0)=" + mv);
                        missingValueTable.put(variableName, mv);
                        invalidDataInfo.setInvalidValues(mv);
                    } else if (recordType2FixedPart1[3] == -2) {
                        dbgLog.fine("mv(-2)=" + mv);
                        // range
                        invalidDataInfo.setInvalidRange(mv);
                    } else if (recordType2FixedPart1[3] == -3) {
                        // mixed case
                        dbgLog.fine("mv(-3)=" + mv);
                        invalidDataInfo.setInvalidRange(mv.subList(0, 2));
                        invalidDataInfo.setInvalidValues(mv.subList(2, 3));
                        missingValueTable.put(variableName, mv.subList(2, 3));
                    }
                    dbgLog.fine("missing value=" + StringUtils.join(missingValueTable.get(variableName), "|"));
                    dbgLog.fine("invalidDataInfo(Numeric):\n" + invalidDataInfo);
                    invalidDataTable.put(variableName, invalidDataInfo);
                } else {
                    // string variable case
                    String[] missingValues = new String[howManyMissingValueUnits];
                    List<String> mv = new ArrayList<String>();
                    int offset_start = 0;
                    int offset_end = LENGTH_SAV_OBS_BLOCK;
                    for (int i = 0; i < howManyMissingValueUnits; i++) {
                        missingValues[i] = StringUtils.stripEnd(new String(Arrays.copyOfRange(missing_value_code_units, offset_start, offset_end), defaultCharSet), " ");
                        dbgLog.fine("missing value=" + missingValues[i] + "<-");
                        offset_start = offset_end;
                        offset_end += LENGTH_SAV_OBS_BLOCK;
                        mv.add(missingValues[i]);
                    }
                    invalidDataInfo.setInvalidValues(mv);
                    missingValueTable.put(variableName, mv);
                    invalidDataTable.put(variableName, invalidDataInfo);
                    dbgLog.fine("missing value(str)=" + StringUtils.join(missingValueTable.get(variableName), "|"));
                    dbgLog.fine("invalidDataInfo(String):\n" + invalidDataInfo);
                }
                // string case
                dbgLog.fine("invalidDataTable:\n" + invalidDataTable);
            }
        // if msv
        } catch (IOException ex) {
            // ex.printStackTrace();
            throw ex;
        } catch (Exception ex) {
            ex.printStackTrace();
        // should we be throwing some exception here?
        }
    }
    if (j != OBSUnitsPerCase) {
        dbgLog.fine("RT2: attention! didn't reach the end of the OBS list!");
        throw new IOException("RT2: didn't reach the end of the OBS list!");
    }
    dbgLog.fine("RT2 metadata-related exit-chores");
    // /smd.getFileInformation().put("varQnty", variableCounter);
    dataTable.setVarQuantity(new Long(variableCounter));
    dbgLog.fine("RT2: varQnty=" + variableCounter);
    // 4.0 Initialize variables:
    List<DataVariable> variableList = new ArrayList<DataVariable>();
    for (int i = 0; i < variableCounter; i++) {
        DataVariable dv = new DataVariable();
        String varName = variableNameList.get(i);
        dbgLog.fine("name: " + varName);
        dv.setName(varName);
        String varLabel = variableLabelMap.get(varName);
        if (varLabel != null && varLabel.length() > 255) {
            // TODO:
            // variable labels will be changed into type 'TEXT' in the
            // database - this will eliminate the 255 char. limit.
            // -- L.A. 4.0 beta11
            dbgLog.fine("Have to truncate label: " + varLabel);
            varLabel = varLabel.substring(0, 255);
        }
        dbgLog.fine("label: " + varLabel);
        dv.setLabel(varLabel);
        dv.setInvalidRanges(new ArrayList<VariableRange>());
        dv.setSummaryStatistics(new ArrayList<SummaryStatistic>());
        dv.setUnf("UNF:6:");
        dv.setCategories(new ArrayList<VariableCategory>());
        variableList.add(dv);
        dv.setFileOrder(i);
        dv.setDataTable(dataTable);
    }
    dataTable.setDataVariables(variableList);
    // /smd.setVariableName(variableNameList.toArray(new String[variableNameList.size()]));
    // /smd.setVariableLabel(variableLabelMap);
    // TODO:
    // figure out what to do with the missing value table!
    // -- 4.0 alpha
    // well, they were used to generate merged summary statistics for
    // the variable. So need to verify what the DDI import was doing
    // with them and replicate the same in 4.0.
    // (add appropriate value labels?)
    // /TODO: 4.0 smd.setMissingValueTable(missingValueTable);
    // /smd.getFileInformation().put("caseWeightVariableName", caseWeightVariableName);
    dbgLog.fine("sumstat:long case=" + Arrays.deepToString(variableTypelList.toArray()));
    dbgLog.fine("RT2: OBSwiseTypelList=" + OBSwiseTypelList);
    dbgLog.fine("decodeRecordType2(): end");
}

Example 3 with SummaryStatistic

use of edu.harvard.iq.dataverse.datavariable.SummaryStatistic in project dataverse by IQSS.

the class DdiExportUtil method createVarDDI.

private static void createVarDDI(XMLStreamWriter xmlw, DataVariable dv) throws XMLStreamException {
    xmlw.writeStartElement("var");
    writeAttribute(xmlw, "ID", "v" + dv.getId().toString());
    writeAttribute(xmlw, "name", dv.getName());
    if (dv.getNumberOfDecimalPoints() != null) {
        writeAttribute(xmlw, "dcml", dv.getNumberOfDecimalPoints().toString());
    }
    if (dv.isOrderedCategorical()) {
        writeAttribute(xmlw, "nature", "ordinal");
    }
    if (dv.getInterval() != null) {
        String interval = dv.getIntervalLabel();
        if (interval != null) {
            writeAttribute(xmlw, "intrvl", interval);
        }
    }
    // location
    xmlw.writeEmptyElement("location");
    if (dv.getFileStartPosition() != null) {
        writeAttribute(xmlw, "StartPos", dv.getFileStartPosition().toString());
    }
    if (dv.getFileEndPosition() != null) {
        writeAttribute(xmlw, "EndPos", dv.getFileEndPosition().toString());
    }
    if (dv.getRecordSegmentNumber() != null) {
        writeAttribute(xmlw, "RecSegNo", dv.getRecordSegmentNumber().toString());
    }
    writeAttribute(xmlw, "fileid", "f" + dv.getDataTable().getDataFile().getId().toString());
    // labl
    if (!StringUtilisEmpty(dv.getLabel())) {
        xmlw.writeStartElement("labl");
        writeAttribute(xmlw, "level", "variable");
        xmlw.writeCharacters(dv.getLabel());
        // labl
        xmlw.writeEndElement();
    }
    // invalrng
    boolean invalrngAdded = false;
    for (VariableRange range : dv.getInvalidRanges()) {
        // if (range.getBeginValueType() != null && range.getBeginValueType().getName().equals(DB_VAR_RANGE_TYPE_POINT)) {
        if (range.getBeginValueType() != null && range.isBeginValueTypePoint()) {
            if (range.getBeginValue() != null) {
                invalrngAdded = checkParentElement(xmlw, "invalrng", invalrngAdded);
                xmlw.writeEmptyElement("item");
                writeAttribute(xmlw, "VALUE", range.getBeginValue());
            }
        } else {
            invalrngAdded = checkParentElement(xmlw, "invalrng", invalrngAdded);
            xmlw.writeEmptyElement("range");
            if (range.getBeginValueType() != null && range.getBeginValue() != null) {
                if (range.isBeginValueTypeMin()) {
                    writeAttribute(xmlw, "min", range.getBeginValue());
                } else if (range.isBeginValueTypeMinExcl()) {
                    writeAttribute(xmlw, "minExclusive", range.getBeginValue());
                }
            }
            if (range.getEndValueType() != null && range.getEndValue() != null) {
                if (range.isEndValueTypeMax()) {
                    writeAttribute(xmlw, "max", range.getEndValue());
                } else if (range.isEndValueTypeMaxExcl()) {
                    writeAttribute(xmlw, "maxExclusive", range.getEndValue());
                }
            }
        }
    }
    if (invalrngAdded) {
        // invalrng
        xmlw.writeEndElement();
    }
    // universe
    if (!StringUtilisEmpty(dv.getUniverse())) {
        xmlw.writeStartElement("universe");
        xmlw.writeCharacters(dv.getUniverse());
        // universe
        xmlw.writeEndElement();
    }
    // sum stats
    for (SummaryStatistic sumStat : dv.getSummaryStatistics()) {
        xmlw.writeStartElement("sumStat");
        if (sumStat.getTypeLabel() != null) {
            writeAttribute(xmlw, "type", sumStat.getTypeLabel());
        } else {
            writeAttribute(xmlw, "type", "unknown");
        }
        xmlw.writeCharacters(sumStat.getValue());
        // sumStat
        xmlw.writeEndElement();
    }
    // categories
    for (VariableCategory cat : dv.getCategories()) {
        xmlw.writeStartElement("catgry");
        if (cat.isMissing()) {
            writeAttribute(xmlw, "missing", "Y");
        }
        // catValu
        xmlw.writeStartElement("catValu");
        xmlw.writeCharacters(cat.getValue());
        // catValu
        xmlw.writeEndElement();
        // label
        if (!StringUtilisEmpty(cat.getLabel())) {
            xmlw.writeStartElement("labl");
            writeAttribute(xmlw, "level", "category");
            xmlw.writeCharacters(cat.getLabel());
            // labl
            xmlw.writeEndElement();
        }
        // catStat
        if (cat.getFrequency() != null) {
            xmlw.writeStartElement("catStat");
            writeAttribute(xmlw, "type", "freq");
            // if frequency is actually a long value, we want to write "100" instead of "100.0"
            if (Math.floor(cat.getFrequency()) == cat.getFrequency()) {
                xmlw.writeCharacters(new Long(cat.getFrequency().longValue()).toString());
            } else {
                xmlw.writeCharacters(cat.getFrequency().toString());
            }
            // catStat
            xmlw.writeEndElement();
        }
        // catgry
        xmlw.writeEndElement();
    }
    // varFormat
    xmlw.writeEmptyElement("varFormat");
    if (dv.isTypeNumeric()) {
        writeAttribute(xmlw, "type", "numeric");
    } else if (dv.isTypeCharacter()) {
        writeAttribute(xmlw, "type", "character");
    } else {
        throw new XMLStreamException("Illegal Variable Format Type!");
    }
    writeAttribute(xmlw, "formatname", dv.getFormat());
    // experiment writeAttribute(xmlw, "schema", dv.getFormatSchema());
    writeAttribute(xmlw, "category", dv.getFormatCategory());
    // notes
    if (dv.getUnf() != null && !"".equals(dv.getUnf())) {
        xmlw.writeStartElement("notes");
        writeAttribute(xmlw, "subject", "Universal Numeric Fingerprint");
        writeAttribute(xmlw, "level", "variable");
        writeAttribute(xmlw, "type", "Dataverse:UNF");
        xmlw.writeCharacters(dv.getUnf());
        // notes
        xmlw.writeEndElement();
    }
    // var
    xmlw.writeEndElement();
}

Example 4 with SummaryStatistic

use of edu.harvard.iq.dataverse.datavariable.SummaryStatistic in project dataverse by IQSS.

the class DDIExportServiceBean method createVarDDI.

private void createVarDDI(XMLStreamWriter xmlw, Set<String> excludedFieldSet, Set<String> includedFieldSet, DataVariable dv) throws XMLStreamException {
    xmlw.writeStartElement("var");
    writeAttribute(xmlw, "ID", "v" + dv.getId().toString());
    writeAttribute(xmlw, "name", dv.getName());
    if (dv.getNumberOfDecimalPoints() != null) {
        writeAttribute(xmlw, "dcml", dv.getNumberOfDecimalPoints().toString());
    }
    if (dv.isOrderedCategorical()) {
        writeAttribute(xmlw, "nature", "ordinal");
    }
    if (dv.getInterval() != null) {
        String interval = dv.getIntervalLabel();
        if (interval != null) {
            writeAttribute(xmlw, "intrvl", interval);
        }
    }
    // location
    if (checkField("location", excludedFieldSet, includedFieldSet)) {
        xmlw.writeEmptyElement("location");
        if (dv.getFileStartPosition() != null) {
            writeAttribute(xmlw, "StartPos", dv.getFileStartPosition().toString());
        }
        if (dv.getFileEndPosition() != null) {
            writeAttribute(xmlw, "EndPos", dv.getFileEndPosition().toString());
        }
        if (dv.getRecordSegmentNumber() != null) {
            writeAttribute(xmlw, "RecSegNo", dv.getRecordSegmentNumber().toString());
        }
        writeAttribute(xmlw, "fileid", "f" + dv.getDataTable().getDataFile().getId().toString());
    }
    // labl
    if (checkField("labl", excludedFieldSet, includedFieldSet)) {
        if (!StringUtilisEmpty(dv.getLabel())) {
            xmlw.writeStartElement("labl");
            writeAttribute(xmlw, "level", "variable");
            xmlw.writeCharacters(dv.getLabel());
            // labl
            xmlw.writeEndElement();
        }
    }
    // invalrng
    if (checkField("invalrng", excludedFieldSet, includedFieldSet)) {
        boolean invalrngAdded = false;
        for (VariableRange range : dv.getInvalidRanges()) {
            // if (range.getBeginValueType() != null && range.getBeginValueType().getName().equals(DB_VAR_RANGE_TYPE_POINT)) {
            if (range.getBeginValueType() != null && range.isBeginValueTypePoint()) {
                if (range.getBeginValue() != null) {
                    invalrngAdded = checkParentElement(xmlw, "invalrng", invalrngAdded);
                    xmlw.writeEmptyElement("item");
                    writeAttribute(xmlw, "VALUE", range.getBeginValue());
                }
            } else {
                invalrngAdded = checkParentElement(xmlw, "invalrng", invalrngAdded);
                xmlw.writeEmptyElement("range");
                if (range.getBeginValueType() != null && range.getBeginValue() != null) {
                    if (range.isBeginValueTypeMin()) {
                        writeAttribute(xmlw, "min", range.getBeginValue());
                    } else if (range.isBeginValueTypeMinExcl()) {
                        writeAttribute(xmlw, "minExclusive", range.getBeginValue());
                    }
                }
                if (range.getEndValueType() != null && range.getEndValue() != null) {
                    if (range.isEndValueTypeMax()) {
                        writeAttribute(xmlw, "max", range.getEndValue());
                    } else if (range.isEndValueTypeMaxExcl()) {
                        writeAttribute(xmlw, "maxExclusive", range.getEndValue());
                    }
                }
            }
        }
        if (invalrngAdded) {
            // invalrng
            xmlw.writeEndElement();
        }
    }
    // universe
    if (checkField("universe", excludedFieldSet, includedFieldSet)) {
        if (!StringUtilisEmpty(dv.getUniverse())) {
            xmlw.writeStartElement("universe");
            xmlw.writeCharacters(dv.getUniverse());
            // universe
            xmlw.writeEndElement();
        }
    }
    // sum stats
    if (checkField("sumStat", excludedFieldSet, includedFieldSet)) {
        for (SummaryStatistic sumStat : dv.getSummaryStatistics()) {
            xmlw.writeStartElement("sumStat");
            if (sumStat.getTypeLabel() != null) {
                writeAttribute(xmlw, "type", sumStat.getTypeLabel());
            } else {
                writeAttribute(xmlw, "type", "unknown");
            }
            xmlw.writeCharacters(sumStat.getValue());
            // sumStat
            xmlw.writeEndElement();
        }
    }
    // categories
    if (checkField("catgry", excludedFieldSet, includedFieldSet)) {
        for (VariableCategory cat : dv.getCategories()) {
            xmlw.writeStartElement("catgry");
            if (cat.isMissing()) {
                writeAttribute(xmlw, "missing", "Y");
            }
            // catValu
            xmlw.writeStartElement("catValu");
            xmlw.writeCharacters(cat.getValue());
            // catValu
            xmlw.writeEndElement();
            // label
            if (!StringUtilisEmpty(cat.getLabel())) {
                xmlw.writeStartElement("labl");
                writeAttribute(xmlw, "level", "category");
                xmlw.writeCharacters(cat.getLabel());
                // labl
                xmlw.writeEndElement();
            }
            // catStat
            if (cat.getFrequency() != null) {
                xmlw.writeStartElement("catStat");
                writeAttribute(xmlw, "type", "freq");
                // if frequency is actually a long value, we want to write "100" instead of "100.0"
                if (Math.floor(cat.getFrequency()) == cat.getFrequency()) {
                    xmlw.writeCharacters(new Long(cat.getFrequency().longValue()).toString());
                } else {
                    xmlw.writeCharacters(cat.getFrequency().toString());
                }
                // catStat
                xmlw.writeEndElement();
            }
            // catgry
            xmlw.writeEndElement();
        }
    }
    // varFormat
    if (checkField("varFormat", excludedFieldSet, includedFieldSet)) {
        xmlw.writeEmptyElement("varFormat");
        if (dv.isTypeNumeric()) {
            writeAttribute(xmlw, "type", "numeric");
        } else if (dv.isTypeCharacter()) {
            writeAttribute(xmlw, "type", "character");
        } else {
            throw new XMLStreamException("Illegal Variable Format Type!");
        }
        writeAttribute(xmlw, "formatname", dv.getFormat());
        // experiment writeAttribute(xmlw, "schema", dv.getFormatSchema());
        writeAttribute(xmlw, "category", dv.getFormatCategory());
    }
    // notes
    if (checkField("unf", excludedFieldSet, includedFieldSet)) {
        xmlw.writeStartElement("notes");
        writeAttribute(xmlw, "subject", "Universal Numeric Fingerprint");
        writeAttribute(xmlw, "level", "variable");
        writeAttribute(xmlw, "type", "VDC:UNF");
        xmlw.writeCharacters(dv.getUnf());
        // notes
        xmlw.writeEndElement();
    }
    // var
    xmlw.writeEndElement();
}

Example 5 with SummaryStatistic

use of edu.harvard.iq.dataverse.datavariable.SummaryStatistic in project dataverse by IQSS.

the class IngestServiceBean method assignContinuousSummaryStatistics.

private void assignContinuousSummaryStatistics(DataVariable variable, double[] sumStats) throws IOException {
    if (sumStats == null || sumStats.length != variableService.summaryStatisticTypes.length) {
        throw new IOException("Wrong number of summary statistics types calculated! (" + sumStats.length + ")");
    }
    for (int j = 0; j < variableService.summaryStatisticTypes.length; j++) {
        SummaryStatistic ss = new SummaryStatistic();
        ss.setTypeByLabel(variableService.summaryStatisticTypes[j]);
        if (!ss.isTypeMode()) {
            ss.setValue((new Double(sumStats[j])).toString());
        } else {
            ss.setValue(".");
        }
        ss.setDataVariable(variable);
        variable.getSummaryStatistics().add(ss);
    }
}