Examples with InvalidData edu.harvard.iq.dataverse.ingest.tabulardata.InvalidData used on opensource projects

Example 1 with InvalidData

use of edu.harvard.iq.dataverse.ingest.tabulardata.InvalidData in project dataverse by IQSS.

the class SAVFileReader method decodeRecordType999.

void decodeRecordType999(BufferedInputStream stream) throws IOException {
    dbgLog.fine("decodeRecordType999(): start");
    try {
        if (stream == null) {
            throw new IllegalArgumentException("RT999: stream == null!");
        }
        // first check the 4-byte header value
        // if (stream.markSupported()){
        stream.mark(1000);
        // }
        // 999.0 check the first 4 bytes
        byte[] headerCodeRt999 = new byte[LENGTH_RECORD_TYPE999_CODE];
        // dbgLog.fine("RT999: stream position="+stream.pos);
        int nbytes_rt999 = stream.read(headerCodeRt999, 0, LENGTH_RECORD_TYPE999_CODE);
        // to-do check against nbytes
        // printHexDump(headerCodeRt999, "RT999 header test");
        ByteBuffer bb_header_code_rt999 = ByteBuffer.wrap(headerCodeRt999, 0, LENGTH_RECORD_TYPE999_CODE);
        if (isLittleEndian) {
            bb_header_code_rt999.order(ByteOrder.LITTLE_ENDIAN);
        }
        int intRT999test = bb_header_code_rt999.getInt();
        dbgLog.fine("header test value: RT999=" + intRT999test);
        if (intRT999test != 999) {
            // if (stream.markSupported()){
            dbgLog.fine("intRT999test failed=" + intRT999test);
            stream.reset();
            throw new IOException("RT999:Header value(999) was not correctly detected:" + intRT999test);
        // }
        }
        // 999.1 check 4-byte integer Filler block
        byte[] length_filler = new byte[LENGTH_RT999_FILLER];
        int nbytes_rt999_1 = stream.read(length_filler, 0, LENGTH_RT999_FILLER);
        // to-do check against nbytes
        // printHexDump(length_how_many_line_bytes, "RT999 how_many_line_bytes");
        ByteBuffer bb_filler = ByteBuffer.wrap(length_filler, 0, LENGTH_RT999_FILLER);
        if (isLittleEndian) {
            bb_filler.order(ByteOrder.LITTLE_ENDIAN);
        }
        int rt999filler = bb_filler.getInt();
        dbgLog.fine("rt999filler=" + rt999filler);
        if (rt999filler == 0) {
            dbgLog.fine("the end of the dictionary section");
        } else {
            throw new IOException("RT999: failed to detect the end mark(0): value=" + rt999filler);
        }
        // missing value processing concerning HIGHEST/LOWEST values
        Set<Map.Entry<String, InvalidData>> msvlc = invalidDataTable.entrySet();
        for (Iterator<Map.Entry<String, InvalidData>> itc = msvlc.iterator(); itc.hasNext(); ) {
            Map.Entry<String, InvalidData> et = itc.next();
            String variable = et.getKey();
            dbgLog.fine("variable=" + variable);
            InvalidData invalidDataInfo = et.getValue();
            if (invalidDataInfo.getInvalidRange() != null && !invalidDataInfo.getInvalidRange().isEmpty()) {
                if (invalidDataInfo.getInvalidRange().get(0).equals(OBSTypeHexValue.get("LOWEST"))) {
                    dbgLog.fine("1st value is LOWEST");
                    invalidDataInfo.getInvalidRange().set(0, "LOWEST");
                } else if (invalidDataInfo.getInvalidRange().get(1).equals(OBSTypeHexValue.get("HIGHEST"))) {
                    dbgLog.fine("2nd value is HIGHEST");
                    invalidDataInfo.getInvalidRange().set(1, "HIGHEST");
                }
            }
        }
        dbgLog.fine("invalidDataTable:\n" + invalidDataTable);
    // TODO: take care of the invalid data! - add the appropriate
    // value labels (?)
    // should it be done here, or at the end of ingest?
    // -- L.A. 4.0 alpha
    // /smd.setInvalidDataTable(invalidDataTable);
    } catch (IOException ex) {
        // exit(1);
        throw ex;
    }
    dbgLog.fine("decodeRecordType999(): end");
}

Example 2 with InvalidData

use of edu.harvard.iq.dataverse.ingest.tabulardata.InvalidData in project dataverse by IQSS.

the class SAVFileReader method decodeRecordType2.

void decodeRecordType2(BufferedInputStream stream) throws IOException {
    dbgLog.fine("decodeRecordType2(): start");
    if (stream == null) {
        throw new IllegalArgumentException("stream == null!");
    }
    Map<String, String> printFormatNameTable = new LinkedHashMap<String, String>();
    Map<String, String> variableLabelMap = new LinkedHashMap<String, String>();
    Map<String, List<String>> missingValueTable = new LinkedHashMap<String, List<String>>();
    List<Integer> printFormatList = new ArrayList<Integer>();
    String caseWeightVariableName = null;
    int caseWeightVariableIndex = 0;
    boolean lastVariableIsExtendable = false;
    boolean extendedVariableMode = false;
    boolean obs255 = false;
    String lastVariableName = null;
    String lastExtendedVariable = null;
    // this field repeats as many as the number of variables in
    // this sav file
    // (note that the above statement is not technically correct, this
    // record repeats not just for every variable in the file, but for
    // every OBS (8 byte unit); i.e., if a string is split into multiple
    // OBS units, each one will have its own RT2 record -- L.A.).
    // Each field constists of a fixed (32-byte) segment and
    // then a few variable segments:
    // if the variable has a label (3rd INT4 set to 1), then there's 4 more
    // bytes specifying the length of the label, and then that many bytes
    // holding the label itself (no more than 256).
    // Then if there are optional missing value units (4th INT4 set to 1)
    // there will be 3 more OBS units attached = 24 extra bytes.
    int variableCounter = 0;
    int obsSeqNumber = 0;
    int j;
    dbgLog.fine("RT2: Reading " + OBSUnitsPerCase + " OBS units.");
    for (j = 0; j < OBSUnitsPerCase; j++) {
        dbgLog.fine("RT2: " + j + "-th RT2 unit is being decoded.");
        // 2.0: read the fixed[=non-optional] 32-byte segment
        byte[] recordType2Fixed = new byte[LENGTH_RECORDTYPE2_FIXED];
        try {
            int nbytes = stream.read(recordType2Fixed, 0, LENGTH_RECORDTYPE2_FIXED);
            if (nbytes == 0) {
                throw new IOException("reading recordType2: no bytes read!");
            }
            int offset = 0;
            // 2.1: create int-view of the bytebuffer for the first 16-byte segment
            int rt2_1st_4_units = 4;
            ByteBuffer[] bb_record_type2_fixed_part1 = new ByteBuffer[rt2_1st_4_units];
            int[] recordType2FixedPart1 = new int[rt2_1st_4_units];
            for (int i = 0; i < rt2_1st_4_units; i++) {
                bb_record_type2_fixed_part1[i] = ByteBuffer.wrap(recordType2Fixed, offset, LENGTH_SAV_INT_BLOCK);
                offset += LENGTH_SAV_INT_BLOCK;
                if (isLittleEndian) {
                    bb_record_type2_fixed_part1[i].order(ByteOrder.LITTLE_ENDIAN);
                }
                recordType2FixedPart1[i] = bb_record_type2_fixed_part1[i].getInt();
            }
            // 1st ([0]) element must be 2 otherwise no longer Record Type 2
            if (recordType2FixedPart1[0] != 2) {
                dbgLog.warning(j + "-th RT header value is no longet RT2! " + recordType2FixedPart1[0]);
                break;
            }
            dbgLog.fine("variable type[must be 2]=" + recordType2FixedPart1[0]);
            // 2.3 variable name: 8 byte(space[x20]-padded)
            // This field is located at the very end of the 32 byte
            // fixed-size RT2 header (bytes 24-31).
            // We are processing it now, so that
            // we can make the decision on whether this variable is part
            // of a compound variable:
            String RawVariableName = getNullStrippedString(new String(Arrays.copyOfRange(recordType2Fixed, 24, (24 + LENGTH_VARIABLE_NAME)), defaultCharSet));
            // offset +=LENGTH_VARIABLE_NAME;
            String variableName = null;
            if (RawVariableName.indexOf(' ') >= 0) {
                variableName = RawVariableName.substring(0, RawVariableName.indexOf(' '));
            } else {
                variableName = RawVariableName;
            }
            // 2nd ([1]) element: numeric variable = 0 :for string variable
            // this block indicates its datum-length, i.e, >0 ;
            // if -1, this RT2 unit is a non-1st RT2 unit for a string variable
            // whose value is longer than 8 character.
            boolean isNumericVariable = false;
            dbgLog.fine("variable type(0: numeric; > 0: String;-1 continue )=" + recordType2FixedPart1[1]);
            // OBSwiseTypelList.add(recordType2FixedPart1[1]);
            int HowManyRt2Units = 1;
            if (recordType2FixedPart1[1] == -1) {
                dbgLog.fine("this RT2 is an 8 bit continuation chunk of an earlier string variable");
                if (obs255) {
                    if (obsSeqNumber < 30) {
                        OBSwiseTypelList.add(recordType2FixedPart1[1]);
                        obsSeqNumber++;
                    } else {
                        OBSwiseTypelList.add(-2);
                        obs255 = false;
                        obsSeqNumber = 0;
                    }
                } else {
                    OBSwiseTypelList.add(recordType2FixedPart1[1]);
                }
                obsNonVariableBlockSet.add(j);
                continue;
            } else if (recordType2FixedPart1[1] == 0) {
                // This is a numeric variable
                extendedVariableMode = false;
                // And as such, it cannot be an extension of a
                // previous, long string variable.
                OBSwiseTypelList.add(recordType2FixedPart1[1]);
                variableCounter++;
                isNumericVariable = true;
                variableTypelList.add(recordType2FixedPart1[1]);
            } else if (recordType2FixedPart1[1] > 0) {
                if (recordType2FixedPart1[1] == 255) {
                    obs255 = true;
                }
                if (lastVariableIsExtendable) {
                    String varNameBase = null;
                    if (lastVariableName.length() > 5) {
                        varNameBase = lastVariableName.substring(0, 5);
                    } else {
                        varNameBase = lastVariableName;
                    }
                    if (extendedVariableMode) {
                        if (variableNameIsAnIncrement(varNameBase, lastExtendedVariable, variableName)) {
                            OBSwiseTypelList.add(-1);
                            lastExtendedVariable = variableName;
                        // OK, we stay in the "extended variable" mode;
                        // but we can't move on to the next OBS (hence the commented out
                        // "continue" below:
                        // continue;
                        // see the next comment below for the explanation.
                        // 
                        // Should we also set "extendable" flag to false at this point
                        // if it's shorter than 255 bytes, i.e. the last extended chunk?
                        } else {
                            extendedVariableMode = false;
                        }
                    } else {
                        if (variableNameIsAnIncrement(varNameBase, variableName)) {
                            OBSwiseTypelList.add(-1);
                            extendedVariableMode = true;
                            dbgLog.fine("RT2: in extended variable mode; variable " + variableName);
                            lastExtendedVariable = variableName;
                        // Before we move on to the next OBS unit, we need to check
                        // if this current extended variable has its own label specified;
                        // If so, we need to determine its length, then read and skip
                        // that many bytes.
                        // Hence the commented out "continue" below:
                        // continue;
                        }
                    }
                }
                if (!extendedVariableMode) {
                    // OK, this is a "real"
                    // string variable, and not a continuation chunk of a compound
                    // string.
                    OBSwiseTypelList.add(recordType2FixedPart1[1]);
                    variableCounter++;
                    if (recordType2FixedPart1[1] == 255) {
                        // This variable is 255 bytes long, i.e. this is
                        // either the single "atomic" variable of the
                        // max allowed size, or it's a 255 byte segment
                        // of a compound variable. So we will check
                        // the next variable and see if it is the continuation
                        // of this one.
                        lastVariableIsExtendable = true;
                    } else {
                        lastVariableIsExtendable = false;
                    }
                    if (recordType2FixedPart1[1] % LENGTH_SAV_OBS_BLOCK == 0) {
                        HowManyRt2Units = recordType2FixedPart1[1] / LENGTH_SAV_OBS_BLOCK;
                    } else {
                        HowManyRt2Units = recordType2FixedPart1[1] / LENGTH_SAV_OBS_BLOCK + 1;
                    }
                    variableTypelList.add(recordType2FixedPart1[1]);
                }
            }
            if (!extendedVariableMode) {
                // Again, we only want to do the following steps for the "real"
                // variables, not the chunks of split mega-variables:
                dbgLog.fine("RT2: HowManyRt2Units for this variable=" + HowManyRt2Units);
                lastVariableName = variableName;
                // caseWeightVariableOBSIndex starts from 1: 0 is used for does-not-exist cases
                if (j == (caseWeightVariableOBSIndex - 1)) {
                    caseWeightVariableName = variableName;
                    // TODO: do we need this "index"? -- 4.0 alpha
                    caseWeightVariableIndex = variableCounter;
                // /smd.setCaseWeightVariableName(caseWeightVariableName);
                // /smd.getFileInformation().put("caseWeightVariableIndex", caseWeightVariableIndex);
                }
                OBSIndexToVariableName.put(j, variableName);
                // dbgLog.fine("\nvariable name="+variableName+"<-");
                dbgLog.fine("RT2: " + j + "-th variable name=" + variableName + "<-");
                dbgLog.fine("RT2: raw variable: " + RawVariableName);
                variableNameList.add(variableName);
            }
            // 3rd ([2]) element: = 1 variable-label block follows; 0 = no label
            // 
            dbgLog.fine("RT: variable label follows?(1:yes; 0: no)=" + recordType2FixedPart1[2]);
            boolean hasVariableLabel = recordType2FixedPart1[2] == 1 ? true : false;
            if ((recordType2FixedPart1[2] != 0) && (recordType2FixedPart1[2] != 1)) {
                throw new IOException("RT2: reading error: value is neither 0 or 1" + recordType2FixedPart1[2]);
            }
            if (hasVariableLabel) {
                byte[] length_variable_label = new byte[4];
                int nbytes_2_4 = stream.read(length_variable_label);
                if (nbytes_2_4 == 0) {
                    throw new IOException("RT 2: error reading recordType2.4: no bytes read!");
                } else {
                    dbgLog.fine("nbytes_2_4=" + nbytes_2_4);
                }
                ByteBuffer bb_length_variable_label = ByteBuffer.wrap(length_variable_label, 0, LENGTH_VARIABLE_LABEL);
                if (isLittleEndian) {
                    bb_length_variable_label.order(ByteOrder.LITTLE_ENDIAN);
                }
                int rawVariableLabelLength = bb_length_variable_label.getInt();
                dbgLog.fine("rawVariableLabelLength=" + rawVariableLabelLength);
                int variableLabelLength = getSAVintAdjustedBlockLength(rawVariableLabelLength);
                dbgLog.fine("RT2: variableLabelLength=" + variableLabelLength);
                // 2.5 [optional]variable label whose length is found at 2.4
                String variableLabel = "";
                if (rawVariableLabelLength > 0) {
                    byte[] variable_label = new byte[variableLabelLength];
                    int nbytes_2_5 = stream.read(variable_label);
                    if (nbytes_2_5 == 0) {
                        throw new IOException("RT 2: error reading recordType2.5: " + variableLabelLength + " bytes requested, no bytes read!");
                    } else {
                        dbgLog.fine("nbytes_2_5=" + nbytes_2_5);
                    }
                    variableLabel = getNullStrippedString(new String(Arrays.copyOfRange(variable_label, 0, rawVariableLabelLength), defaultCharSet));
                    dbgLog.fine("RT2: variableLabel=" + variableLabel + "<-");
                    dbgLog.fine(variableName + " => " + variableLabel);
                } else {
                    dbgLog.fine("RT2: defaulting to empty variable label.");
                }
                if (!extendedVariableMode) {
                    // We only have any use for this label if it's a "real" variable.
                    // Thinking about it, it doesn't make much sense for the "fake"
                    // variables that are actually chunks of large strings to store
                    // their own labels. But in some files they do. Then failing to read
                    // the bytes would result in getting out of sync with the RT record
                    // borders. So we always read the bytes, but only use them for
                    // the real variable entries.
                    /*String variableLabel = new String(Arrays.copyOfRange(variable_label,
                                0, rawVariableLabelLength),"US-ASCII");*/
                    variableLabelMap.put(variableName, variableLabel);
                }
            }
            if (extendedVariableMode) {
                // that's how SPSS stores them.
                continue;
            }
            // 4th ([3]) element: Missing value type code
            // 0[none], 1, 2, 3 [point-type],-2[range], -3 [range type+ point]
            dbgLog.fine("RT: missing value unit follows?(if 0, none)=" + recordType2FixedPart1[3]);
            boolean hasMissingValues = (validMissingValueCodeSet.contains(recordType2FixedPart1[3]) && (recordType2FixedPart1[3] != 0)) ? true : false;
            InvalidData invalidDataInfo = null;
            if (recordType2FixedPart1[3] != 0) {
                invalidDataInfo = new InvalidData(recordType2FixedPart1[3]);
                dbgLog.fine("RT: missing value type=" + invalidDataInfo.getType());
            }
            // 2.2: print/write formats: 4-byte each = 8 bytes
            byte[] printFormt = Arrays.copyOfRange(recordType2Fixed, offset, offset + LENGTH_PRINT_FORMAT_CODE);
            dbgLog.fine("printFrmt=" + new String(Hex.encodeHex(printFormt)));
            offset += LENGTH_PRINT_FORMAT_CODE;
            int formatCode = isLittleEndian ? printFormt[2] : printFormt[1];
            int formatWidth = isLittleEndian ? printFormt[1] : printFormt[2];
            // TODO:
            // What should we be doing with these "format decimal positions"
            // in 4.0?
            // -- L.A. 4.0 alpha
            int formatDecimalPointPosition = isLittleEndian ? printFormt[0] : printFormt[3];
            dbgLog.fine("RT2: format code{5=F, 1=A[String]}=" + formatCode);
            formatDecimalPointPositionList.add(formatDecimalPointPosition);
            if (!SPSSConstants.FORMAT_CODE_TABLE_SAV.containsKey(formatCode)) {
                throw new IOException("Unknown format code was found = " + formatCode);
            } else {
                printFormatList.add(formatCode);
            }
            byte[] writeFormt = Arrays.copyOfRange(recordType2Fixed, offset, offset + LENGTH_WRITE_FORMAT_CODE);
            dbgLog.fine("RT2: writeFrmt=" + new String(Hex.encodeHex(writeFormt)));
            if (writeFormt[3] != 0x00) {
                dbgLog.fine("byte-order(write format): reversal required");
            }
            offset += LENGTH_WRITE_FORMAT_CODE;
            if (!SPSSConstants.ORDINARY_FORMAT_CODE_SET.contains(formatCode)) {
                StringBuilder sb = new StringBuilder(SPSSConstants.FORMAT_CODE_TABLE_SAV.get(formatCode) + formatWidth);
                if (formatDecimalPointPosition > 0) {
                    sb.append("." + formatDecimalPointPosition);
                }
                dbgLog.fine("formattable[i] = " + variableName + " -> " + sb.toString());
                printFormatNameTable.put(variableName, sb.toString());
            }
            printFormatTable.put(variableName, SPSSConstants.FORMAT_CODE_TABLE_SAV.get(formatCode));
            if (hasMissingValues) {
                dbgLog.fine("RT2: decoding missing value: type=" + recordType2FixedPart1[3]);
                int howManyMissingValueUnits = missingValueCodeUnits.get(recordType2FixedPart1[3]);
                // int howManyMissingValueUnits = recordType2FixedPart1[3] > 0 ? recordType2FixedPart1[3] :  0;
                dbgLog.fine("RT2: howManyMissingValueUnits=" + howManyMissingValueUnits);
                byte[] missing_value_code_units = new byte[LENGTH_SAV_OBS_BLOCK * howManyMissingValueUnits];
                int nbytes_2_6 = stream.read(missing_value_code_units);
                if (nbytes_2_6 == 0) {
                    throw new IOException("RT 2: reading recordType2.6: no byte was read");
                } else {
                    dbgLog.fine("nbytes_2_6=" + nbytes_2_6);
                }
                if (isNumericVariable) {
                    double[] missingValues = new double[howManyMissingValueUnits];
                    // List<String> mvp = new ArrayList<String>();
                    List<String> mv = new ArrayList<String>();
                    ByteBuffer[] bb_missig_value_code = new ByteBuffer[howManyMissingValueUnits];
                    int offset_start = 0;
                    for (int i = 0; i < howManyMissingValueUnits; i++) {
                        bb_missig_value_code[i] = ByteBuffer.wrap(missing_value_code_units, offset_start, LENGTH_SAV_OBS_BLOCK);
                        offset_start += LENGTH_SAV_OBS_BLOCK;
                        if (isLittleEndian) {
                            bb_missig_value_code[i].order(ByteOrder.LITTLE_ENDIAN);
                        }
                        ByteBuffer temp = bb_missig_value_code[i].duplicate();
                        missingValues[i] = bb_missig_value_code[i].getDouble();
                        if (Double.toHexString(missingValues[i]).equals("-0x1.ffffffffffffep1023")) {
                            dbgLog.fine("1st value is LOWEST");
                            mv.add(Double.toHexString(missingValues[i]));
                        } else if (Double.valueOf(missingValues[i]).equals(Double.MAX_VALUE)) {
                            dbgLog.fine("2nd value is HIGHEST");
                            mv.add(Double.toHexString(missingValues[i]));
                        } else {
                            mv.add(doubleNumberFormatter.format(missingValues[i]));
                        }
                        dbgLog.fine(i + "-th missing value=" + Double.toHexString(missingValues[i]));
                    }
                    dbgLog.fine("variableName=" + variableName);
                    if (recordType2FixedPart1[3] > 0) {
                        // point cases only
                        dbgLog.fine("mv(>0)=" + mv);
                        missingValueTable.put(variableName, mv);
                        invalidDataInfo.setInvalidValues(mv);
                    } else if (recordType2FixedPart1[3] == -2) {
                        dbgLog.fine("mv(-2)=" + mv);
                        // range
                        invalidDataInfo.setInvalidRange(mv);
                    } else if (recordType2FixedPart1[3] == -3) {
                        // mixed case
                        dbgLog.fine("mv(-3)=" + mv);
                        invalidDataInfo.setInvalidRange(mv.subList(0, 2));
                        invalidDataInfo.setInvalidValues(mv.subList(2, 3));
                        missingValueTable.put(variableName, mv.subList(2, 3));
                    }
                    dbgLog.fine("missing value=" + StringUtils.join(missingValueTable.get(variableName), "|"));
                    dbgLog.fine("invalidDataInfo(Numeric):\n" + invalidDataInfo);
                    invalidDataTable.put(variableName, invalidDataInfo);
                } else {
                    // string variable case
                    String[] missingValues = new String[howManyMissingValueUnits];
                    List<String> mv = new ArrayList<String>();
                    int offset_start = 0;
                    int offset_end = LENGTH_SAV_OBS_BLOCK;
                    for (int i = 0; i < howManyMissingValueUnits; i++) {
                        missingValues[i] = StringUtils.stripEnd(new String(Arrays.copyOfRange(missing_value_code_units, offset_start, offset_end), defaultCharSet), " ");
                        dbgLog.fine("missing value=" + missingValues[i] + "<-");
                        offset_start = offset_end;
                        offset_end += LENGTH_SAV_OBS_BLOCK;
                        mv.add(missingValues[i]);
                    }
                    invalidDataInfo.setInvalidValues(mv);
                    missingValueTable.put(variableName, mv);
                    invalidDataTable.put(variableName, invalidDataInfo);
                    dbgLog.fine("missing value(str)=" + StringUtils.join(missingValueTable.get(variableName), "|"));
                    dbgLog.fine("invalidDataInfo(String):\n" + invalidDataInfo);
                }
                // string case
                dbgLog.fine("invalidDataTable:\n" + invalidDataTable);
            }
        // if msv
        } catch (IOException ex) {
            // ex.printStackTrace();
            throw ex;
        } catch (Exception ex) {
            ex.printStackTrace();
        // should we be throwing some exception here?
        }
    }
    if (j != OBSUnitsPerCase) {
        dbgLog.fine("RT2: attention! didn't reach the end of the OBS list!");
        throw new IOException("RT2: didn't reach the end of the OBS list!");
    }
    dbgLog.fine("RT2 metadata-related exit-chores");
    // /smd.getFileInformation().put("varQnty", variableCounter);
    dataTable.setVarQuantity(new Long(variableCounter));
    dbgLog.fine("RT2: varQnty=" + variableCounter);
    // 4.0 Initialize variables:
    List<DataVariable> variableList = new ArrayList<DataVariable>();
    for (int i = 0; i < variableCounter; i++) {
        DataVariable dv = new DataVariable();
        String varName = variableNameList.get(i);
        dbgLog.fine("name: " + varName);
        dv.setName(varName);
        String varLabel = variableLabelMap.get(varName);
        if (varLabel != null && varLabel.length() > 255) {
            // TODO:
            // variable labels will be changed into type 'TEXT' in the
            // database - this will eliminate the 255 char. limit.
            // -- L.A. 4.0 beta11
            dbgLog.fine("Have to truncate label: " + varLabel);
            varLabel = varLabel.substring(0, 255);
        }
        dbgLog.fine("label: " + varLabel);
        dv.setLabel(varLabel);
        dv.setInvalidRanges(new ArrayList<VariableRange>());
        dv.setSummaryStatistics(new ArrayList<SummaryStatistic>());
        dv.setUnf("UNF:6:");
        dv.setCategories(new ArrayList<VariableCategory>());
        variableList.add(dv);
        dv.setFileOrder(i);
        dv.setDataTable(dataTable);
    }
    dataTable.setDataVariables(variableList);
    // /smd.setVariableName(variableNameList.toArray(new String[variableNameList.size()]));
    // /smd.setVariableLabel(variableLabelMap);
    // TODO:
    // figure out what to do with the missing value table!
    // -- 4.0 alpha
    // well, they were used to generate merged summary statistics for
    // the variable. So need to verify what the DDI import was doing
    // with them and replicate the same in 4.0.
    // (add appropriate value labels?)
    // /TODO: 4.0 smd.setMissingValueTable(missingValueTable);
    // /smd.getFileInformation().put("caseWeightVariableName", caseWeightVariableName);
    dbgLog.fine("sumstat:long case=" + Arrays.deepToString(variableTypelList.toArray()));
    dbgLog.fine("RT2: OBSwiseTypelList=" + OBSwiseTypelList);
    dbgLog.fine("decodeRecordType2(): end");
}

Example 3 with InvalidData

use of edu.harvard.iq.dataverse.ingest.tabulardata.InvalidData in project dataverse by IQSS.

the class PORFileReader method processMissingValueData.

private void processMissingValueData() {
    /*

         POR's missing-value storage differs form the counterpart of SAV;
         this method transforms the POR-native storage to the SAV-type
         after this process, missingValueTable contains point-type
         missing values for later catStat/sumStat processing;
         range and mixed type cases are stored in invalidDataTable

         missingValueCodeTable=
            {VAR1=[9], VAR2=[A], VAR3=[9, 8], VAR4=[A, 8],
             VAR5=[8, 8, 8], VAR6=[B], VAR7=[B, 8]}

         missingValueTable=
            {VAR1=[-1], VAR2=[-1], VAR3=[-2, -1], VAR4=[-1, -2],
             VAR5=[-1, -2, -3], VAR6=[-2, -1], VAR7=[-3, -2, -1]}


         missingValueTable={VAR1=[], VAR2=[], VAR3=[-1], VAR4=[-2],
             VAR5=[-1, -2, -3], VAR6=[], VAR7=[-2]}

         */
    dbgLog.fine("missingValueCodeTable=" + missingValueCodeTable);
    Set<Map.Entry<String, List<String>>> msvlc = missingValueCodeTable.entrySet();
    for (Map.Entry<String, List<String>> et : msvlc) {
        String variable = et.getKey();
        dbgLog.fine("variable=" + variable);
        List<String> codeList = et.getValue();
        List<String> valueList = missingValueTable.get(variable);
        dbgLog.fine("codeList=" + codeList);
        dbgLog.fine("valueList=" + valueList);
        int type;
        InvalidData invalidDataInfo = null;
        if (valueList.size() == 3) {
            if (codeList.get(0).equals("8") && codeList.get(1).equals("8") && codeList.get(2).equals("8")) {
                type = 3;
                invalidDataInfo = new InvalidData(type);
                invalidDataInfo.setInvalidValues(valueList);
            } else if (codeList.get(0).equals("9") && codeList.get(1).equals("8")) {
                type = -3;
                invalidDataInfo = new InvalidData(type);
                invalidDataInfo.setInvalidValues(valueList.subList(2, 3));
                invalidDataInfo.setInvalidRange(valueList.subList(0, 2));
            } else if (codeList.get(0).equals("A") && codeList.get(1).equals("8")) {
                type = -3;
                invalidDataInfo = new InvalidData(type);
                invalidDataInfo.setInvalidValues(valueList.subList(2, 3));
                invalidDataInfo.setInvalidRange(valueList.subList(0, 2));
            } else if (codeList.get(0).equals("B") && codeList.get(1).equals("8")) {
                type = -3;
                invalidDataInfo = new InvalidData(type);
                invalidDataInfo.setInvalidValues(valueList.subList(2, 3));
                invalidDataInfo.setInvalidRange(valueList.subList(0, 2));
            } else {
                dbgLog.severe("unkown missing-value combination(3 values)");
            }
        } else if (valueList.size() == 2) {
            if (codeList.get(0).equals("8") && codeList.get(1).equals("8")) {
                type = 2;
                invalidDataInfo = new InvalidData(type);
                invalidDataInfo.setInvalidValues(valueList);
            } else if (codeList.get(0).equals("9")) {
                type = -2;
                invalidDataInfo = new InvalidData(type);
                invalidDataInfo.setInvalidRange(valueList.subList(0, 2));
            } else if (codeList.get(0).equals("A")) {
                type = -2;
                invalidDataInfo = new InvalidData(type);
                invalidDataInfo.setInvalidRange(valueList.subList(0, 2));
            } else if (codeList.get(0).equals("B")) {
                type = -2;
                invalidDataInfo = new InvalidData(type);
                invalidDataInfo.setInvalidRange(valueList.subList(0, 2));
            } else {
                dbgLog.severe("unknown missing value combination(2 values)");
            }
        } else if (valueList.size() == 1) {
            if (codeList.get(0).equals("8")) {
                type = 1;
                invalidDataInfo = new InvalidData(type);
                invalidDataInfo.setInvalidValues(valueList);
            } else {
                dbgLog.severe("unknown missing value combination(2 values)");
            }
        }
        invalidDataTable.put(variable, invalidDataInfo);
    }
    dbgLog.fine("invalidDataTable=" + invalidDataTable);
    Set<Map.Entry<String, List<String>>> msvl = missingValueTable.entrySet();
    for (Map.Entry<String, List<String>> et : msvl) {
        String variable = et.getKey();
        List<String> valueList = et.getValue();
        List<String> codeList = missingValueCodeTable.get(variable);
        dbgLog.finer("var=" + variable + "\tvalue=" + valueList + "\t code" + codeList);
        List<String> temp = new ArrayList<>();
        for (int j = 0; j < codeList.size(); j++) {
            if (codeList.get(j).equals("8")) {
                temp.add(valueList.get(j));
            }
        }
        missingValueTable.put(variable, temp);
    }
    dbgLog.fine("missingValueTable=" + missingValueTable);
}