use of org.apache.wink.json4j.JSONObject in project incubator-systemml by apache.
the class DataExpression method checkHasMatrixMarketFormat.
public boolean checkHasMatrixMarketFormat(String inputFileName, String mtdFileName, boolean conditional) {
// Check the MTD file exists. if there is an MTD file, return false.
JSONObject mtdObject = readMetadataFile(mtdFileName, conditional);
if (mtdObject != null)
return false;
if (MapReduceTool.existsFileOnHDFS(inputFileName) && !MapReduceTool.isDirectory(inputFileName)) {
BufferedReader in = null;
try {
Path path = new Path(inputFileName);
FileSystem fs = IOUtilFunctions.getFileSystem(path);
in = new BufferedReader(new InputStreamReader(fs.open(path)));
String headerLine = new String("");
if (in.ready())
headerLine = in.readLine();
return (headerLine != null && headerLine.startsWith("%%"));
} catch (Exception ex) {
throw new LanguageException("Failed to read mtd file.", ex);
} finally {
IOUtilFunctions.closeSilently(in);
}
}
return false;
}
use of org.apache.wink.json4j.JSONObject in project incubator-systemml by apache.
the class DataExpression method parseMetaDataFileParameters.
@SuppressWarnings("unchecked")
private void parseMetaDataFileParameters(String mtdFileName, JSONObject configObject, boolean conditional) {
for (Object obj : configObject.entrySet()) {
Entry<Object, Object> e = (Entry<Object, Object>) obj;
Object key = e.getKey();
Object val = e.getValue();
boolean isValidName = false;
for (String paramName : READ_VALID_MTD_PARAM_NAMES) {
if (paramName.equals(key))
isValidName = true;
}
if (!isValidName) {
// wrong parameters always rejected
raiseValidateError("MTD file " + mtdFileName + " contains invalid parameter name: " + key, false);
}
// if the read method parameter is a constant, then verify value matches MTD metadata file
if (getVarParam(key.toString()) != null && (getVarParam(key.toString()) instanceof ConstIdentifier) && !getVarParam(key.toString()).toString().equalsIgnoreCase(val.toString())) {
raiseValidateError("Parameter '" + key.toString() + "' has conflicting values in metadata and read statement. MTD file value: '" + val.toString() + "'. Read statement value: '" + getVarParam(key.toString()) + "'.", conditional);
} else {
// if the read method does not specify parameter value, then add MTD metadata file value to parameter list
if (getVarParam(key.toString()) == null) {
if ((!key.toString().equalsIgnoreCase(DESCRIPTIONPARAM)) && (!key.toString().equalsIgnoreCase(AUTHORPARAM)) && (!key.toString().equalsIgnoreCase(CREATEDPARAM))) {
StringIdentifier strId = new StringIdentifier(val.toString(), this);
if (key.toString().equalsIgnoreCase(DELIM_HAS_HEADER_ROW) || key.toString().equalsIgnoreCase(DELIM_FILL) || key.toString().equalsIgnoreCase(DELIM_SPARSE)) {
// parse these parameters as boolean values
BooleanIdentifier boolId = null;
if (strId.toString().equalsIgnoreCase("true")) {
boolId = new BooleanIdentifier(true, this);
} else if (strId.toString().equalsIgnoreCase("false")) {
boolId = new BooleanIdentifier(false, this);
} else {
raiseValidateError("Invalid value provided for '" + DELIM_HAS_HEADER_ROW + "' in metadata file '" + mtdFileName + "'. " + "Must be either TRUE or FALSE.", conditional);
}
removeVarParam(key.toString());
addVarParam(key.toString(), boolId);
} else if (key.toString().equalsIgnoreCase(DELIM_FILL_VALUE)) {
// parse these parameters as numeric values
DoubleIdentifier doubleId = new DoubleIdentifier(Double.parseDouble(strId.toString()), this);
removeVarParam(key.toString());
addVarParam(key.toString(), doubleId);
} else if (key.toString().equalsIgnoreCase(DELIM_NA_STRINGS)) {
String naStrings = null;
if (val instanceof String) {
naStrings = val.toString();
} else {
StringBuilder sb = new StringBuilder();
JSONArray valarr = (JSONArray) val;
for (int naid = 0; naid < valarr.size(); naid++) {
sb.append((String) valarr.get(naid));
if (naid < valarr.size() - 1)
sb.append(DELIM_NA_STRING_SEP);
}
naStrings = sb.toString();
}
StringIdentifier sid = new StringIdentifier(naStrings, this);
removeVarParam(key.toString());
addVarParam(key.toString(), sid);
} else {
// by default, treat a parameter as a string
addVarParam(key.toString(), strId);
}
}
}
}
}
}
use of org.apache.wink.json4j.JSONObject in project incubator-systemml by apache.
the class DataExpression method readMetadataFile.
public JSONObject readMetadataFile(String filename, boolean conditional) {
JSONObject retVal = null;
boolean exists = MapReduceTool.existsFileOnHDFS(filename);
boolean isDir = MapReduceTool.isDirectory(filename);
// CASE: filename is a directory -- process as a directory
if (exists && isDir) {
retVal = new JSONObject();
for (FileStatus stat : MapReduceTool.getDirectoryListing(filename)) {
// gives directory name
Path childPath = stat.getPath();
if (!childPath.getName().startsWith("part"))
continue;
BufferedReader br = null;
try {
FileSystem fs = IOUtilFunctions.getFileSystem(childPath);
br = new BufferedReader(new InputStreamReader(fs.open(childPath)));
JSONObject childObj = JSONHelper.parse(br);
for (Object obj : childObj.entrySet()) {
@SuppressWarnings("unchecked") Entry<Object, Object> e = (Entry<Object, Object>) obj;
Object key = e.getKey();
Object val = e.getValue();
retVal.put(key, val);
}
} catch (Exception e) {
raiseValidateError("for MTD file in directory, error parting part of MTD file with path " + childPath.toString() + ": " + e.getMessage(), conditional);
} finally {
IOUtilFunctions.closeSilently(br);
}
}
} else // CASE: filename points to a file
if (exists) {
BufferedReader br = null;
try {
Path path = new Path(filename);
FileSystem fs = IOUtilFunctions.getFileSystem(path);
br = new BufferedReader(new InputStreamReader(fs.open(path)));
retVal = new JSONObject(br);
} catch (Exception e) {
raiseValidateError("error parsing MTD file with path " + filename + ": " + e.getMessage(), conditional);
} finally {
IOUtilFunctions.closeSilently(br);
}
}
return retVal;
}
use of org.apache.wink.json4j.JSONObject in project incubator-systemml by apache.
the class AutomatedTestBase method readDMLMetaDataFile.
public static MatrixCharacteristics readDMLMetaDataFile(String fileName) {
try {
String fname = baseDirectory + OUTPUT_DIR + fileName + ".mtd";
JSONObject meta = new DataExpression().readMetadataFile(fname, false);
long rlen = Long.parseLong(meta.get(DataExpression.READROWPARAM).toString());
long clen = Long.parseLong(meta.get(DataExpression.READCOLPARAM).toString());
return new MatrixCharacteristics(rlen, clen, -1, -1);
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
use of org.apache.wink.json4j.JSONObject in project incubator-systemml by apache.
the class DecoderFactory method createDecoder.
@SuppressWarnings("unchecked")
public static Decoder createDecoder(String spec, String[] colnames, ValueType[] schema, FrameBlock meta) {
Decoder decoder = null;
try {
// parse transform specification
JSONObject jSpec = new JSONObject(spec);
List<Decoder> ldecoders = new ArrayList<>();
// create decoders 'recode', 'dummy' and 'pass-through'
List<Integer> rcIDs = Arrays.asList(ArrayUtils.toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfUtils.TXMETHOD_RECODE)));
List<Integer> dcIDs = Arrays.asList(ArrayUtils.toObject(TfMetaUtils.parseJsonIDList(jSpec, colnames, TfUtils.TXMETHOD_DUMMYCODE)));
rcIDs = new ArrayList<Integer>(CollectionUtils.union(rcIDs, dcIDs));
List<Integer> ptIDs = new ArrayList<Integer>(CollectionUtils.subtract(UtilFunctions.getSeqList(1, meta.getNumColumns(), 1), rcIDs));
// create default schema if unspecified (with double columns for pass-through)
if (schema == null) {
schema = UtilFunctions.nCopies(meta.getNumColumns(), ValueType.STRING);
for (Integer col : ptIDs) schema[col - 1] = ValueType.DOUBLE;
}
if (!dcIDs.isEmpty()) {
ldecoders.add(new DecoderDummycode(schema, ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0]))));
}
if (!rcIDs.isEmpty()) {
ldecoders.add(new DecoderRecode(schema, !dcIDs.isEmpty(), ArrayUtils.toPrimitive(rcIDs.toArray(new Integer[0]))));
}
if (!ptIDs.isEmpty()) {
ldecoders.add(new DecoderPassThrough(schema, ArrayUtils.toPrimitive(ptIDs.toArray(new Integer[0])), ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0]))));
}
// create composite decoder of all created decoders
// and initialize with given meta data (recode, dummy, bin)
decoder = new DecoderComposite(schema, ldecoders);
decoder.setColnames(colnames);
if (meta != null)
decoder.initMetaData(meta);
} catch (Exception ex) {
throw new DMLRuntimeException(ex);
}
return decoder;
}
Aggregations