Examples with DataExpression org.apache.sysml.parser.DataExpression used on opensource projects

Example 1 with DataExpression

use of org.apache.sysml.parser.DataExpression in project incubator-systemml by apache.

the class Connection method readDoubleMatrix.

// //////////////////////////////////////////
// Read matrices
// //////////////////////////////////////////
/**
 * Reads an input matrix in arbitrary format from HDFS into a dense double array.
 * NOTE: this call currently only supports default configurations for CSV.
 *
 * @param fname the filename of the input matrix
 * @return matrix as a two-dimensional double array
 * @throws IOException if IOException occurs
 */
public double[][] readDoubleMatrix(String fname) throws IOException {
    try {
        // read json meta data
        String fnamemtd = DataExpression.getMTDFileName(fname);
        JSONObject jmtd = new DataExpression().readMetadataFile(fnamemtd, false);
        // parse json meta data
        long rows = jmtd.getLong(DataExpression.READROWPARAM);
        long cols = jmtd.getLong(DataExpression.READCOLPARAM);
        int brlen = jmtd.containsKey(DataExpression.ROWBLOCKCOUNTPARAM) ? jmtd.getInt(DataExpression.ROWBLOCKCOUNTPARAM) : -1;
        int bclen = jmtd.containsKey(DataExpression.COLUMNBLOCKCOUNTPARAM) ? jmtd.getInt(DataExpression.COLUMNBLOCKCOUNTPARAM) : -1;
        long nnz = jmtd.containsKey(DataExpression.READNUMNONZEROPARAM) ? jmtd.getLong(DataExpression.READNUMNONZEROPARAM) : -1;
        String format = jmtd.getString(DataExpression.FORMAT_TYPE);
        InputInfo iinfo = InputInfo.stringExternalToInputInfo(format);
        // read matrix file
        return readDoubleMatrix(fname, iinfo, rows, cols, brlen, bclen, nnz);
    } catch (Exception ex) {
        throw new IOException(ex);
    }
}

Example 2 with DataExpression

use of org.apache.sysml.parser.DataExpression in project incubator-systemml by apache.

the class Connection method readStringFrame.

// //////////////////////////////////////////
// Read frames
// //////////////////////////////////////////
/**
 * Reads an input frame in arbitrary format from HDFS into a dense string array.
 * NOTE: this call currently only supports default configurations for CSV.
 *
 * @param fname the filename of the input frame
 * @return frame as a two-dimensional string array
 * @throws IOException if IOException occurs
 */
public String[][] readStringFrame(String fname) throws IOException {
    try {
        // read json meta data
        String fnamemtd = DataExpression.getMTDFileName(fname);
        JSONObject jmtd = new DataExpression().readMetadataFile(fnamemtd, false);
        // parse json meta data
        long rows = jmtd.getLong(DataExpression.READROWPARAM);
        long cols = jmtd.getLong(DataExpression.READCOLPARAM);
        String format = jmtd.getString(DataExpression.FORMAT_TYPE);
        InputInfo iinfo = InputInfo.stringExternalToInputInfo(format);
        // read frame file
        return readStringFrame(fname, iinfo, rows, cols);
    } catch (Exception ex) {
        throw new IOException(ex);
    }
}

Example 3 with DataExpression

use of org.apache.sysml.parser.DataExpression in project incubator-systemml by apache.

the class CommonSyntacticValidator method setOutputStatement.

protected void setOutputStatement(ParserRuleContext ctx, ArrayList<ParameterExpression> paramExpression, StatementInfo info) {
    if (paramExpression.size() < 2) {
        notifyErrorListeners("incorrect usage of write function (at least 2 arguments required)", ctx.start);
        return;
    }
    if (paramExpression.get(0).getExpr() instanceof DataIdentifier) {
        HashMap<String, Expression> varParams = new HashMap<>();
        varParams.put(DataExpression.IO_FILENAME, paramExpression.get(1).getExpr());
        for (int i = 2; i < paramExpression.size(); i++) {
            // DataExpression.FORMAT_TYPE, DataExpression.DELIM_DELIMITER, DataExpression.DELIM_HAS_HEADER_ROW,  DataExpression.DELIM_SPARSE
            varParams.put(paramExpression.get(i).getName(), paramExpression.get(i).getExpr());
        }
        DataExpression dataExpression = new DataExpression(ctx, DataOp.WRITE, varParams, currentFile);
        info.stmt = new OutputStatement(ctx, (DataIdentifier) paramExpression.get(0).getExpr(), DataOp.WRITE, currentFile);
        ((OutputStatement) info.stmt).setExprParams(dataExpression);
    } else {
        notifyErrorListeners("incorrect usage of write function", ctx.start);
    }
}

Example 4 with DataExpression

use of org.apache.sysml.parser.DataExpression in project incubator-systemml by apache.

the class CommonSyntacticValidator method buildForBuiltInFunction.

/**
 * Creates a builtin function expression.
 *
 * @param ctx antlr rule context
 * @param functionName Name of the builtin function
 * @param paramExpressions Array of parameter names and values
 * @return expression if found otherwise null
 */
protected Expression buildForBuiltInFunction(ParserRuleContext ctx, String functionName, ArrayList<ParameterExpression> paramExpressions) {
    // Double verification: verify passed function name is a (non-parameterized) built-in function.
    try {
        if (functions.contains(functionName)) {
            // It is a user function definition (which takes precedence if name same as built-in)
            return null;
        }
        Expression lsf = handleLanguageSpecificFunction(ctx, functionName, paramExpressions);
        if (lsf != null) {
            setFileLineColumn(lsf, ctx);
            return lsf;
        }
        BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(ctx, functionName, paramExpressions, currentFile);
        if (bife != null) {
            // It is a builtin function
            return bife;
        }
        ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(ctx, functionName, paramExpressions, currentFile);
        if (pbife != null) {
            // It is a parameterized builtin function
            return pbife;
        }
        // built-in read, rand ...
        DataExpression dbife = DataExpression.getDataExpression(ctx, functionName, paramExpressions, currentFile, errorListener);
        if (dbife != null) {
            return dbife;
        }
    } catch (Exception e) {
        notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
    }
    return null;
}

Example 5 with DataExpression

use of org.apache.sysml.parser.DataExpression in project incubator-systemml by apache.

the class AutomatedTestBase method readDMLMetaDataFile.

public static MatrixCharacteristics readDMLMetaDataFile(String fileName) {
    try {
        String fname = baseDirectory + OUTPUT_DIR + fileName + ".mtd";
        JSONObject meta = new DataExpression().readMetadataFile(fname, false);
        long rlen = Long.parseLong(meta.get(DataExpression.READROWPARAM).toString());
        long clen = Long.parseLong(meta.get(DataExpression.READCOLPARAM).toString());
        return new MatrixCharacteristics(rlen, clen, -1, -1);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}