Examples with ValueType org.apache.sysml.parser.Expression.ValueType used on opensource projects

Example 46 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class FrameRDDConverterUtils method convertFrameSchemaToDFSchema.

/**
 * This function will convert Frame schema into DataFrame schema
 *
 * @param fschema frame schema
 * @param containsID true if contains ID column
 * @return Spark StructType of StructFields representing schema
 */
public static StructType convertFrameSchemaToDFSchema(ValueType[] fschema, boolean containsID) {
    // generate the schema based on the string of schema
    List<StructField> fields = new ArrayList<>();
    // add id column type
    if (containsID)
        fields.add(DataTypes.createStructField(RDDConverterUtils.DF_ID_COLUMN, DataTypes.DoubleType, true));
    // add remaining types
    int col = 1;
    for (ValueType schema : fschema) {
        DataType dt = null;
        switch(schema) {
            case STRING:
                dt = DataTypes.StringType;
                break;
            case DOUBLE:
                dt = DataTypes.DoubleType;
                break;
            case INT:
                dt = DataTypes.LongType;
                break;
            case BOOLEAN:
                dt = DataTypes.BooleanType;
                break;
            default:
                dt = DataTypes.StringType;
                LOG.warn("Using default type String for " + schema.toString());
        }
        fields.add(DataTypes.createStructField("C" + col++, dt, true));
    }
    return DataTypes.createStructType(fields);
}

Example 47 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class FrameReaderBinaryBlock method readFrameFromHDFS.

@Override
public final FrameBlock readFrameFromHDFS(String fname, ValueType[] schema, String[] names, long rlen, long clen) throws IOException, DMLRuntimeException {
    // allocate output frame block
    ValueType[] lschema = createOutputSchema(schema, clen);
    String[] lnames = createOutputNames(names, clen);
    FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);
    // prepare file access
    JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
    Path path = new Path(fname);
    FileSystem fs = IOUtilFunctions.getFileSystem(path, job);
    // check existence and non-empty file
    checkValidInputFile(fs, path);
    // core read (sequential/parallel)
    readBinaryBlockFrameFromHDFS(path, job, fs, ret, rlen, clen);
    return ret;
}

Example 48 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class OperationsOnMatrixValues method performSlice.

/**
 * This function will get slice of the input frame block overlapping in overall slice(Range), slice has requested for.
 *
 * @param in ?
 * @param ixrange index range
 * @param brlen number of rows in a block
 * @param bclen number of columns in a block
 * @param outlist list of pairs of frame blocks
 */
public static void performSlice(Pair<Long, FrameBlock> in, IndexRange ixrange, int brlen, int bclen, ArrayList<Pair<Long, FrameBlock>> outlist) {
    long index = in.getKey();
    FrameBlock block = in.getValue();
    // Get Block indexes (rows and columns boundaries)
    long cellIndexTopRow = index;
    long cellIndexBottomRow = index + block.getNumRows() - 1;
    long cellIndexLeftCol = 1;
    long cellIndexRightCol = block.getNumColumns();
    // Calculate block boundaries with range of slice to be performed (Global index)
    long cellIndexOverlapTop = Math.max(cellIndexTopRow, ixrange.rowStart);
    long cellIndexOverlapBottom = Math.min(cellIndexBottomRow, ixrange.rowEnd);
    long cellIndexOverlapLeft = Math.max(cellIndexLeftCol, ixrange.colStart);
    long cellIndexOverlapRight = Math.min(cellIndexRightCol, ixrange.colEnd);
    // check if block is outside the indexing range
    if (cellIndexOverlapTop > cellIndexOverlapBottom || cellIndexOverlapLeft > cellIndexOverlapRight) {
        return;
    }
    // Create IndexRange for the slice to be performed on this block.
    IndexRange tmpRange = new IndexRange(cellIndexOverlapTop - index, cellIndexOverlapBottom - index, cellIndexOverlapLeft - 1, cellIndexOverlapRight - 1);
    // Get Top Row and Left column cutting point.
    int rowCut = (int) (ixrange.rowStart - index);
    // Get indices for result block
    long resultBlockIndexTop = UtilFunctions.computeBlockIndex(cellIndexOverlapTop, brlen);
    long resultBlockIndexBottom = UtilFunctions.computeBlockIndex(cellIndexOverlapBottom, brlen);
    // allocate space for the output value
    for (long r = resultBlockIndexTop; r <= resultBlockIndexBottom; r++) {
        ValueType[] schema = Arrays.copyOfRange(block.getSchema(), (int) tmpRange.colStart, (int) tmpRange.colEnd + 1);
        long iResultIndex = Math.max(((r - 1) * brlen - ixrange.rowStart + 1), 0);
        Pair<Long, FrameBlock> out = new Pair<>(new Long(iResultIndex + 1), new FrameBlock(schema));
        outlist.add(out);
    }
    // execute actual slice operation
    block.slice(outlist, tmpRange, rowCut);
}

Example 49 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class GenerateClassesForMLContext method generateFunctionCallMethod.

/**
 * Obtain method for invoking a script function.
 *
 * @param scriptFilePath
 *            the path to a script file
 * @param fs
 *            a SystemML function statement
 * @param dmlFunctionCall
 *            a string representing the invocation of a script function
 * @return string representation of a method that performs a function call
 */
public static String generateFunctionCallMethod(String scriptFilePath, FunctionStatement fs, String dmlFunctionCall) {
    createFunctionOutputClass(scriptFilePath, fs);
    StringBuilder sb = new StringBuilder();
    sb.append("public ");
    // begin return type
    ArrayList<DataIdentifier> oparams = fs.getOutputParams();
    if (oparams.size() == 0) {
        sb.append("void");
    } else if (oparams.size() == 1) {
        // if 1 output, no need to encapsulate it, so return the output
        // directly
        DataIdentifier oparam = oparams.get(0);
        String type = getParamTypeAsString(oparam);
        sb.append(type);
    } else {
        String fullFunctionOutputClassName = getFullFunctionOutputClassName(scriptFilePath, fs);
        sb.append(fullFunctionOutputClassName);
    }
    sb.append(" ");
    // end return type
    sb.append(fs.getName());
    sb.append("(");
    ArrayList<DataIdentifier> inputParams = fs.getInputParams();
    for (int i = 0; i < inputParams.size(); i++) {
        if (i > 0) {
            sb.append(", ");
        }
        DataIdentifier inputParam = inputParams.get(i);
        /*
			 * Note: Using Object is currently preferrable to using
			 * datatype/valuetype to explicitly set the input type to
			 * Integer/Double/Boolean/String since Object allows the automatic
			 * handling of things such as automatic conversions from longs to
			 * ints.
			 */
        sb.append("Object ");
        sb.append(inputParam.getName());
    }
    sb.append(") {\n");
    sb.append("String scriptString = \"" + dmlFunctionCall + "\";\n");
    sb.append("org.apache.sysml.api.mlcontext.Script script = new org.apache.sysml.api.mlcontext.Script(scriptString);\n");
    if ((inputParams.size() > 0) || (oparams.size() > 0)) {
        sb.append("script");
    }
    for (int i = 0; i < inputParams.size(); i++) {
        DataIdentifier inputParam = inputParams.get(i);
        String name = inputParam.getName();
        sb.append(".in(\"" + name + "\", " + name + ")");
    }
    for (int i = 0; i < oparams.size(); i++) {
        DataIdentifier outputParam = oparams.get(i);
        String name = outputParam.getName();
        sb.append(".out(\"" + name + "\")");
    }
    if ((inputParams.size() > 0) || (oparams.size() > 0)) {
        sb.append(";\n");
    }
    sb.append("org.apache.sysml.api.mlcontext.MLResults results = script.execute();\n");
    if (oparams.size() == 0) {
        sb.append("return;\n");
    } else if (oparams.size() == 1) {
        DataIdentifier o = oparams.get(0);
        DataType dt = o.getDataType();
        ValueType vt = o.getValueType();
        if ((dt == DataType.SCALAR) && (vt == ValueType.INT)) {
            sb.append("long res = results.getLong(\"" + o.getName() + "\");\nreturn res;\n");
        } else if ((dt == DataType.SCALAR) && (vt == ValueType.DOUBLE)) {
            sb.append("double res = results.getDouble(\"" + o.getName() + "\");\nreturn res;\n");
        } else if ((dt == DataType.SCALAR) && (vt == ValueType.BOOLEAN)) {
            sb.append("boolean res = results.getBoolean(\"" + o.getName() + "\");\nreturn res;\n");
        } else if ((dt == DataType.SCALAR) && (vt == ValueType.STRING)) {
            sb.append("String res = results.getString(\"" + o.getName() + "\");\nreturn res;\n");
        } else if (dt == DataType.MATRIX) {
            sb.append("org.apache.sysml.api.mlcontext.Matrix res = results.getMatrix(\"" + o.getName() + "\");\nreturn res;\n");
        } else if (dt == DataType.FRAME) {
            sb.append("org.apache.sysml.api.mlcontext.Frame res = results.getFrame(\"" + o.getName() + "\");\nreturn res;\n");
        }
    } else {
        for (int i = 0; i < oparams.size(); i++) {
            DataIdentifier outputParam = oparams.get(i);
            String name = outputParam.getName().toLowerCase();
            String type = getParamTypeAsString(outputParam);
            DataType dt = outputParam.getDataType();
            ValueType vt = outputParam.getValueType();
            if ((dt == DataType.SCALAR) && (vt == ValueType.INT)) {
                sb.append(type + " " + name + " = results.getLong(\"" + outputParam.getName() + "\");\n");
            } else if ((dt == DataType.SCALAR) && (vt == ValueType.DOUBLE)) {
                sb.append(type + " " + name + " = results.getDouble(\"" + outputParam.getName() + "\");\n");
            } else if ((dt == DataType.SCALAR) && (vt == ValueType.BOOLEAN)) {
                sb.append(type + " " + name + " = results.getBoolean(\"" + outputParam.getName() + "\");\n");
            } else if ((dt == DataType.SCALAR) && (vt == ValueType.STRING)) {
                sb.append(type + " " + name + " = results.getString(\"" + outputParam.getName() + "\");\n");
            } else if (dt == DataType.MATRIX) {
                sb.append(type + " " + name + " = results.getMatrix(\"" + outputParam.getName() + "\");\n");
            } else if (dt == DataType.FRAME) {
                sb.append(type + " " + name + " = results.getFrame(\"" + outputParam.getName() + "\");\n");
            }
        }
        String ffocn = getFullFunctionOutputClassName(scriptFilePath, fs);
        sb.append(ffocn + " res = new " + ffocn + "(");
        for (int i = 0; i < oparams.size(); i++) {
            if (i > 0) {
                sb.append(", ");
            }
            DataIdentifier outputParam = oparams.get(i);
            String name = outputParam.getName().toLowerCase();
            sb.append(name);
        }
        sb.append(");\nreturn res;\n");
    }
    sb.append("}\n");
    return sb.toString();
}

Example 50 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class EncoderPassThrough method apply.

@Override
public MatrixBlock apply(FrameBlock in, MatrixBlock out) {
    for (int j = 0; j < _colList.length; j++) {
        int col = _colList[j] - 1;
        ValueType vt = in.getSchema()[col];
        for (int i = 0; i < in.getNumRows(); i++) {
            Object val = in.get(i, col);
            out.quickSetValue(i, col, (val == null || (vt == ValueType.STRING && val.toString().isEmpty())) ? Double.NaN : UtilFunctions.objectToDouble(vt, val));
        }
    }
    return out;
}