Search in sources :

Example 11 with JSONObject

use of org.apache.wink.json4j.JSONObject in project incubator-systemml by apache.

the class DataExpression method checkHasMatrixMarketFormat.

public boolean checkHasMatrixMarketFormat(String inputFileName, String mtdFileName, boolean conditional) {
    // Check the MTD file exists. if there is an MTD file, return false.
    JSONObject mtdObject = readMetadataFile(mtdFileName, conditional);
    if (mtdObject != null)
        return false;
    if (MapReduceTool.existsFileOnHDFS(inputFileName) && !MapReduceTool.isDirectory(inputFileName)) {
        BufferedReader in = null;
        try {
            Path path = new Path(inputFileName);
            FileSystem fs = IOUtilFunctions.getFileSystem(path);
            in = new BufferedReader(new InputStreamReader(fs.open(path)));
            String headerLine = new String("");
            if (in.ready())
                headerLine = in.readLine();
            return (headerLine != null && headerLine.startsWith("%%"));
        } catch (Exception ex) {
            throw new LanguageException("Failed to read mtd file.", ex);
        } finally {
            IOUtilFunctions.closeSilently(in);
        }
    }
    return false;
}
Also used : Path(org.apache.hadoop.fs.Path) JSONObject(org.apache.wink.json4j.JSONObject) InputStreamReader(java.io.InputStreamReader) FileSystem(org.apache.hadoop.fs.FileSystem) BufferedReader(java.io.BufferedReader) IOException(java.io.IOException)

Example 12 with JSONObject

use of org.apache.wink.json4j.JSONObject in project incubator-systemml by apache.

the class DataExpression method parseMetaDataFileParameters.

@SuppressWarnings("unchecked")
private void parseMetaDataFileParameters(String mtdFileName, JSONObject configObject, boolean conditional) {
    for (Object obj : configObject.entrySet()) {
        Entry<Object, Object> e = (Entry<Object, Object>) obj;
        Object key = e.getKey();
        Object val = e.getValue();
        boolean isValidName = false;
        for (String paramName : READ_VALID_MTD_PARAM_NAMES) {
            if (paramName.equals(key))
                isValidName = true;
        }
        if (!isValidName) {
            // wrong parameters always rejected
            raiseValidateError("MTD file " + mtdFileName + " contains invalid parameter name: " + key, false);
        }
        // if the read method parameter is a constant, then verify value matches MTD metadata file
        if (getVarParam(key.toString()) != null && (getVarParam(key.toString()) instanceof ConstIdentifier) && !getVarParam(key.toString()).toString().equalsIgnoreCase(val.toString())) {
            raiseValidateError("Parameter '" + key.toString() + "' has conflicting values in metadata and read statement. MTD file value: '" + val.toString() + "'. Read statement value: '" + getVarParam(key.toString()) + "'.", conditional);
        } else {
            // if the read method does not specify parameter value, then add MTD metadata file value to parameter list
            if (getVarParam(key.toString()) == null) {
                if ((!key.toString().equalsIgnoreCase(DESCRIPTIONPARAM)) && (!key.toString().equalsIgnoreCase(AUTHORPARAM)) && (!key.toString().equalsIgnoreCase(CREATEDPARAM))) {
                    StringIdentifier strId = new StringIdentifier(val.toString(), this);
                    if (key.toString().equalsIgnoreCase(DELIM_HAS_HEADER_ROW) || key.toString().equalsIgnoreCase(DELIM_FILL) || key.toString().equalsIgnoreCase(DELIM_SPARSE)) {
                        // parse these parameters as boolean values
                        BooleanIdentifier boolId = null;
                        if (strId.toString().equalsIgnoreCase("true")) {
                            boolId = new BooleanIdentifier(true, this);
                        } else if (strId.toString().equalsIgnoreCase("false")) {
                            boolId = new BooleanIdentifier(false, this);
                        } else {
                            raiseValidateError("Invalid value provided for '" + DELIM_HAS_HEADER_ROW + "' in metadata file '" + mtdFileName + "'. " + "Must be either TRUE or FALSE.", conditional);
                        }
                        removeVarParam(key.toString());
                        addVarParam(key.toString(), boolId);
                    } else if (key.toString().equalsIgnoreCase(DELIM_FILL_VALUE)) {
                        // parse these parameters as numeric values
                        DoubleIdentifier doubleId = new DoubleIdentifier(Double.parseDouble(strId.toString()), this);
                        removeVarParam(key.toString());
                        addVarParam(key.toString(), doubleId);
                    } else if (key.toString().equalsIgnoreCase(DELIM_NA_STRINGS)) {
                        String naStrings = null;
                        if (val instanceof String) {
                            naStrings = val.toString();
                        } else {
                            StringBuilder sb = new StringBuilder();
                            JSONArray valarr = (JSONArray) val;
                            for (int naid = 0; naid < valarr.size(); naid++) {
                                sb.append((String) valarr.get(naid));
                                if (naid < valarr.size() - 1)
                                    sb.append(DELIM_NA_STRING_SEP);
                            }
                            naStrings = sb.toString();
                        }
                        StringIdentifier sid = new StringIdentifier(naStrings, this);
                        removeVarParam(key.toString());
                        addVarParam(key.toString(), sid);
                    } else {
                        // by default, treat a parameter as a string
                        addVarParam(key.toString(), strId);
                    }
                }
            }
        }
    }
}
Also used : Entry(java.util.Map.Entry) JSONArray(org.apache.wink.json4j.JSONArray) JSONObject(org.apache.wink.json4j.JSONObject)

Example 13 with JSONObject

use of org.apache.wink.json4j.JSONObject in project incubator-systemml by apache.

the class DataExpression method readMetadataFile.

public JSONObject readMetadataFile(String filename, boolean conditional) {
    JSONObject retVal = null;
    boolean exists = MapReduceTool.existsFileOnHDFS(filename);
    boolean isDir = MapReduceTool.isDirectory(filename);
    // CASE: filename is a directory -- process as a directory
    if (exists && isDir) {
        retVal = new JSONObject();
        for (FileStatus stat : MapReduceTool.getDirectoryListing(filename)) {
            // gives directory name
            Path childPath = stat.getPath();
            if (!childPath.getName().startsWith("part"))
                continue;
            BufferedReader br = null;
            try {
                FileSystem fs = IOUtilFunctions.getFileSystem(childPath);
                br = new BufferedReader(new InputStreamReader(fs.open(childPath)));
                JSONObject childObj = JSONHelper.parse(br);
                for (Object obj : childObj.entrySet()) {
                    @SuppressWarnings("unchecked") Entry<Object, Object> e = (Entry<Object, Object>) obj;
                    Object key = e.getKey();
                    Object val = e.getValue();
                    retVal.put(key, val);
                }
            } catch (Exception e) {
                raiseValidateError("for MTD file in directory, error parting part of MTD file with path " + childPath.toString() + ": " + e.getMessage(), conditional);
            } finally {
                IOUtilFunctions.closeSilently(br);
            }
        }
    } else // CASE: filename points to a file
    if (exists) {
        BufferedReader br = null;
        try {
            Path path = new Path(filename);
            FileSystem fs = IOUtilFunctions.getFileSystem(path);
            br = new BufferedReader(new InputStreamReader(fs.open(path)));
            retVal = new JSONObject(br);
        } catch (Exception e) {
            raiseValidateError("error parsing MTD file with path " + filename + ": " + e.getMessage(), conditional);
        } finally {
            IOUtilFunctions.closeSilently(br);
        }
    }
    return retVal;
}
Also used : Path(org.apache.hadoop.fs.Path) Entry(java.util.Map.Entry) FileStatus(org.apache.hadoop.fs.FileStatus) JSONObject(org.apache.wink.json4j.JSONObject) InputStreamReader(java.io.InputStreamReader) FileSystem(org.apache.hadoop.fs.FileSystem) BufferedReader(java.io.BufferedReader) JSONObject(org.apache.wink.json4j.JSONObject) IOException(java.io.IOException)

Example 14 with JSONObject

use of org.apache.wink.json4j.JSONObject in project incubator-systemml by apache.

the class AutomatedTestBase method readDMLMetaDataFile.

public static MatrixCharacteristics readDMLMetaDataFile(String fileName) {
    try {
        String fname = baseDirectory + OUTPUT_DIR + fileName + ".mtd";
        JSONObject meta = new DataExpression().readMetadataFile(fname, false);
        long rlen = Long.parseLong(meta.get(DataExpression.READROWPARAM).toString());
        long clen = Long.parseLong(meta.get(DataExpression.READCOLPARAM).toString());
        return new MatrixCharacteristics(rlen, clen, -1, -1);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}
Also used : DataExpression(org.apache.sysml.parser.DataExpression) DMLRuntimeException(org.apache.sysml.runtime.DMLRuntimeException) JSONObject(org.apache.wink.json4j.JSONObject) DMLRuntimeException(org.apache.sysml.runtime.DMLRuntimeException) IOException(java.io.IOException) MatrixCharacteristics(org.apache.sysml.runtime.matrix.MatrixCharacteristics)

Example 15 with JSONObject

use of org.apache.wink.json4j.JSONObject in project incubator-systemml by apache.

the class DecoderFactory method createDecoder.

@SuppressWarnings("unchecked")
public static Decoder createDecoder(String spec, String[] colnames, ValueType[] schema, FrameBlock meta) {
    Decoder decoder = null;
    try {
        // parse transform specification
        JSONObject jSpec = new JSONObject(spec);
        List<Decoder> ldecoders = new ArrayList<>();
        // create decoders 'recode', 'dummy' and 'pass-through'
        List<Integer> rcIDs = Arrays.asList(ArrayUtils.toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfUtils.TXMETHOD_RECODE)));
        List<Integer> dcIDs = Arrays.asList(ArrayUtils.toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfUtils.TXMETHOD_DUMMYCODE)));
        rcIDs = new ArrayList<Integer>(CollectionUtils.union(rcIDs, dcIDs));
        List<Integer> ptIDs = new ArrayList<Integer>(CollectionUtils.subtract(UtilFunctions.getSeqList(1, meta.getNumColumns(), 1), rcIDs));
        // create default schema if unspecified (with double columns for pass-through)
        if (schema == null) {
            schema = UtilFunctions.nCopies(meta.getNumColumns(), ValueType.STRING);
            for (Integer col : ptIDs) schema[col - 1] = ValueType.DOUBLE;
        }
        if (!dcIDs.isEmpty()) {
            ldecoders.add(new DecoderDummycode(schema, ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0]))));
        }
        if (!rcIDs.isEmpty()) {
            ldecoders.add(new DecoderRecode(schema, !dcIDs.isEmpty(), ArrayUtils.toPrimitive(rcIDs.toArray(new Integer[0]))));
        }
        if (!ptIDs.isEmpty()) {
            ldecoders.add(new DecoderPassThrough(schema, ArrayUtils.toPrimitive(ptIDs.toArray(new Integer[0])), ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0]))));
        }
        // create composite decoder of all created decoders
        // and initialize with given meta data (recode, dummy, bin)
        decoder = new DecoderComposite(schema, ldecoders);
        decoder.setColnames(colnames);
        if (meta != null)
            decoder.initMetaData(meta);
    } catch (Exception ex) {
        throw new DMLRuntimeException(ex);
    }
    return decoder;
}
Also used : ArrayList(java.util.ArrayList) DMLRuntimeException(org.apache.sysml.runtime.DMLRuntimeException) DMLRuntimeException(org.apache.sysml.runtime.DMLRuntimeException) JSONObject(org.apache.wink.json4j.JSONObject)

Aggregations

JSONObject (org.apache.wink.json4j.JSONObject)24 DMLRuntimeException (org.apache.sysml.runtime.DMLRuntimeException)14 IOException (java.io.IOException)12 Path (org.apache.hadoop.fs.Path)8 JSONArray (org.apache.wink.json4j.JSONArray)8 BufferedReader (java.io.BufferedReader)5 InputStreamReader (java.io.InputStreamReader)5 FileSystem (org.apache.hadoop.fs.FileSystem)5 DMLException (org.apache.sysml.api.DMLException)4 DataExpression (org.apache.sysml.parser.DataExpression)4 LanguageException (org.apache.sysml.parser.LanguageException)4 ParseException (org.apache.sysml.parser.ParseException)4 BufferedWriter (java.io.BufferedWriter)3 OutputStreamWriter (java.io.OutputStreamWriter)3 ArrayList (java.util.ArrayList)3 Entry (java.util.Map.Entry)2 FrameObject (org.apache.sysml.runtime.controlprogram.caching.FrameObject)2 MatrixObject (org.apache.sysml.runtime.controlprogram.caching.MatrixObject)2 KahanObject (org.apache.sysml.runtime.instructions.cp.KahanObject)2 RDDObject (org.apache.sysml.runtime.instructions.spark.data.RDDObject)2