Examples with JSONObject org.apache.wink.json4j.JSONObject used on opensource projects

Example 11 with JSONObject

use of org.apache.wink.json4j.JSONObject in project incubator-systemml by apache.

the class DataExpression method checkHasMatrixMarketFormat.

public boolean checkHasMatrixMarketFormat(String inputFileName, String mtdFileName, boolean conditional) {
    // Check the MTD file exists. if there is an MTD file, return false.
    JSONObject mtdObject = readMetadataFile(mtdFileName, conditional);
    if (mtdObject != null)
        return false;
    if (MapReduceTool.existsFileOnHDFS(inputFileName) && !MapReduceTool.isDirectory(inputFileName)) {
        BufferedReader in = null;
        try {
            Path path = new Path(inputFileName);
            FileSystem fs = IOUtilFunctions.getFileSystem(path);
            in = new BufferedReader(new InputStreamReader(fs.open(path)));
            String headerLine = new String("");
            if (in.ready())
                headerLine = in.readLine();
            return (headerLine != null && headerLine.startsWith("%%"));
        } catch (Exception ex) {
            throw new LanguageException("Failed to read mtd file.", ex);
        } finally {
            IOUtilFunctions.closeSilently(in);
        }
    }
    return false;
}

Example 12 with JSONObject

use of org.apache.wink.json4j.JSONObject in project incubator-systemml by apache.

the class DataExpression method parseMetaDataFileParameters.

@SuppressWarnings("unchecked")
private void parseMetaDataFileParameters(String mtdFileName, JSONObject configObject, boolean conditional) {
    for (Object obj : configObject.entrySet()) {
        Entry<Object, Object> e = (Entry<Object, Object>) obj;
        Object key = e.getKey();
        Object val = e.getValue();
        boolean isValidName = false;
        for (String paramName : READ_VALID_MTD_PARAM_NAMES) {
            if (paramName.equals(key))
                isValidName = true;
        }
        if (!isValidName) {
            // wrong parameters always rejected
            raiseValidateError("MTD file " + mtdFileName + " contains invalid parameter name: " + key, false);
        }
        // if the read method parameter is a constant, then verify value matches MTD metadata file
        if (getVarParam(key.toString()) != null && (getVarParam(key.toString()) instanceof ConstIdentifier) && !getVarParam(key.toString()).toString().equalsIgnoreCase(val.toString())) {
            raiseValidateError("Parameter '" + key.toString() + "' has conflicting values in metadata and read statement. MTD file value: '" + val.toString() + "'. Read statement value: '" + getVarParam(key.toString()) + "'.", conditional);
        } else {
            // if the read method does not specify parameter value, then add MTD metadata file value to parameter list
            if (getVarParam(key.toString()) == null) {
                if ((!key.toString().equalsIgnoreCase(DESCRIPTIONPARAM)) && (!key.toString().equalsIgnoreCase(AUTHORPARAM)) && (!key.toString().equalsIgnoreCase(CREATEDPARAM))) {
                    StringIdentifier strId = new StringIdentifier(val.toString(), this);
                    if (key.toString().equalsIgnoreCase(DELIM_HAS_HEADER_ROW) || key.toString().equalsIgnoreCase(DELIM_FILL) || key.toString().equalsIgnoreCase(DELIM_SPARSE)) {
                        // parse these parameters as boolean values
                        BooleanIdentifier boolId = null;
                        if (strId.toString().equalsIgnoreCase("true")) {
                            boolId = new BooleanIdentifier(true, this);
                        } else if (strId.toString().equalsIgnoreCase("false")) {
                            boolId = new BooleanIdentifier(false, this);
                        } else {
                            raiseValidateError("Invalid value provided for '" + DELIM_HAS_HEADER_ROW + "' in metadata file '" + mtdFileName + "'. " + "Must be either TRUE or FALSE.", conditional);
                        }
                        removeVarParam(key.toString());
                        addVarParam(key.toString(), boolId);
                    } else if (key.toString().equalsIgnoreCase(DELIM_FILL_VALUE)) {
                        // parse these parameters as numeric values
                        DoubleIdentifier doubleId = new DoubleIdentifier(Double.parseDouble(strId.toString()), this);
                        removeVarParam(key.toString());
                        addVarParam(key.toString(), doubleId);
                    } else if (key.toString().equalsIgnoreCase(DELIM_NA_STRINGS)) {
                        String naStrings = null;
                        if (val instanceof String) {
                            naStrings = val.toString();
                        } else {
                            StringBuilder sb = new StringBuilder();
                            JSONArray valarr = (JSONArray) val;
                            for (int naid = 0; naid < valarr.size(); naid++) {
                                sb.append((String) valarr.get(naid));
                                if (naid < valarr.size() - 1)
                                    sb.append(DELIM_NA_STRING_SEP);
                            }
                            naStrings = sb.toString();
                        }
                        StringIdentifier sid = new StringIdentifier(naStrings, this);
                        removeVarParam(key.toString());
                        addVarParam(key.toString(), sid);
                    } else {
                        // by default, treat a parameter as a string
                        addVarParam(key.toString(), strId);
                    }
                }
            }
        }
    }
}

Example 13 with JSONObject

use of org.apache.wink.json4j.JSONObject in project incubator-systemml by apache.

the class DataExpression method readMetadataFile.

public JSONObject readMetadataFile(String filename, boolean conditional) {
    JSONObject retVal = null;
    boolean exists = MapReduceTool.existsFileOnHDFS(filename);
    boolean isDir = MapReduceTool.isDirectory(filename);
    // CASE: filename is a directory -- process as a directory
    if (exists && isDir) {
        retVal = new JSONObject();
        for (FileStatus stat : MapReduceTool.getDirectoryListing(filename)) {
            // gives directory name
            Path childPath = stat.getPath();
            if (!childPath.getName().startsWith("part"))
                continue;
            BufferedReader br = null;
            try {
                FileSystem fs = IOUtilFunctions.getFileSystem(childPath);
                br = new BufferedReader(new InputStreamReader(fs.open(childPath)));
                JSONObject childObj = JSONHelper.parse(br);
                for (Object obj : childObj.entrySet()) {
                    @SuppressWarnings("unchecked") Entry<Object, Object> e = (Entry<Object, Object>) obj;
                    Object key = e.getKey();
                    Object val = e.getValue();
                    retVal.put(key, val);
                }
            } catch (Exception e) {
                raiseValidateError("for MTD file in directory, error parting part of MTD file with path " + childPath.toString() + ": " + e.getMessage(), conditional);
            } finally {
                IOUtilFunctions.closeSilently(br);
            }
        }
    } else // CASE: filename points to a file
    if (exists) {
        BufferedReader br = null;
        try {
            Path path = new Path(filename);
            FileSystem fs = IOUtilFunctions.getFileSystem(path);
            br = new BufferedReader(new InputStreamReader(fs.open(path)));
            retVal = new JSONObject(br);
        } catch (Exception e) {
            raiseValidateError("error parsing MTD file with path " + filename + ": " + e.getMessage(), conditional);
        } finally {
            IOUtilFunctions.closeSilently(br);
        }
    }
    return retVal;
}

Example 14 with JSONObject

use of org.apache.wink.json4j.JSONObject in project incubator-systemml by apache.

the class AutomatedTestBase method readDMLMetaDataFile.

public static MatrixCharacteristics readDMLMetaDataFile(String fileName) {
    try {
        String fname = baseDirectory + OUTPUT_DIR + fileName + ".mtd";
        JSONObject meta = new DataExpression().readMetadataFile(fname, false);
        long rlen = Long.parseLong(meta.get(DataExpression.READROWPARAM).toString());
        long clen = Long.parseLong(meta.get(DataExpression.READCOLPARAM).toString());
        return new MatrixCharacteristics(rlen, clen, -1, -1);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}

Example 15 with JSONObject

use of org.apache.wink.json4j.JSONObject in project incubator-systemml by apache.

the class DecoderFactory method createDecoder.

@SuppressWarnings("unchecked")
public static Decoder createDecoder(String spec, String[] colnames, ValueType[] schema, FrameBlock meta) {
    Decoder decoder = null;
    try {
        // parse transform specification
        JSONObject jSpec = new JSONObject(spec);
        List<Decoder> ldecoders = new ArrayList<>();
        // create decoders 'recode', 'dummy' and 'pass-through'
        List<Integer> rcIDs = Arrays.asList(ArrayUtils.toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfUtils.TXMETHOD_RECODE)));
        List<Integer> dcIDs = Arrays.asList(ArrayUtils.toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfUtils.TXMETHOD_DUMMYCODE)));
        rcIDs = new ArrayList<Integer>(CollectionUtils.union(rcIDs, dcIDs));
        List<Integer> ptIDs = new ArrayList<Integer>(CollectionUtils.subtract(UtilFunctions.getSeqList(1, meta.getNumColumns(), 1), rcIDs));
        // create default schema if unspecified (with double columns for pass-through)
        if (schema == null) {
            schema = UtilFunctions.nCopies(meta.getNumColumns(), ValueType.STRING);
            for (Integer col : ptIDs) schema[col - 1] = ValueType.DOUBLE;
        }
        if (!dcIDs.isEmpty()) {
            ldecoders.add(new DecoderDummycode(schema, ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0]))));
        }
        if (!rcIDs.isEmpty()) {
            ldecoders.add(new DecoderRecode(schema, !dcIDs.isEmpty(), ArrayUtils.toPrimitive(rcIDs.toArray(new Integer[0]))));
        }
        if (!ptIDs.isEmpty()) {
            ldecoders.add(new DecoderPassThrough(schema, ArrayUtils.toPrimitive(ptIDs.toArray(new Integer[0])), ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0]))));
        }
        // create composite decoder of all created decoders
        // and initialize with given meta data (recode, dummy, bin)
        decoder = new DecoderComposite(schema, ldecoders);
        decoder.setColnames(colnames);
        if (meta != null)
            decoder.initMetaData(meta);
    } catch (Exception ex) {
        throw new DMLRuntimeException(ex);
    }
    return decoder;
}