Examples with ValueType org.apache.sysml.parser.Expression.ValueType used on opensource projects

Example 51 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class TfMetaUtils method convertToTransformMetaDataFrame.

/**
 * Converts transform meta data into an in-memory FrameBlock object.
 *
 * @param rows number of rows
 * @param colnames column names
 * @param rcIDs recode IDs
 * @param binIDs binning IDs
 * @param meta ?
 * @param mvmeta ?
 * @return frame block
 * @throws IOException if IOException occurs
 */
private static FrameBlock convertToTransformMetaDataFrame(int rows, String[] colnames, List<Integer> rcIDs, List<Integer> binIDs, HashMap<String, String> meta, HashMap<String, String> mvmeta) throws IOException {
    // create frame block w/ pure string schema
    ValueType[] schema = UtilFunctions.nCopies(colnames.length, ValueType.STRING);
    FrameBlock ret = new FrameBlock(schema, colnames);
    ret.ensureAllocatedColumns(rows);
    // encode recode maps (recoding/dummycoding) into frame
    for (Integer colID : rcIDs) {
        String name = colnames[colID - 1];
        String map = meta.get(name);
        if (map == null)
            throw new IOException("Recode map for column '" + name + "' (id=" + colID + ") not existing.");
        InputStream is = new ByteArrayInputStream(map.getBytes("UTF-8"));
        BufferedReader br = new BufferedReader(new InputStreamReader(is));
        Pair<String, String> pair = new Pair<>();
        String line;
        int rpos = 0;
        while ((line = br.readLine()) != null) {
            DecoderRecode.parseRecodeMapEntry(line, pair);
            String tmp = pair.getKey() + Lop.DATATYPE_PREFIX + pair.getValue();
            ret.set(rpos++, colID - 1, tmp);
        }
        ret.getColumnMetadata(colID - 1).setNumDistinct((long) rpos);
    }
    // encode bin maps (binning) into frame
    for (Integer colID : binIDs) {
        String name = colnames[colID - 1];
        String map = meta.get(name);
        if (map == null)
            throw new IOException("Binning map for column '" + name + "' (id=" + colID + ") not existing.");
        String[] fields = map.split(TfUtils.TXMTD_SEP);
        double min = UtilFunctions.parseToDouble(fields[1]);
        double binwidth = UtilFunctions.parseToDouble(fields[3]);
        int nbins = UtilFunctions.parseToInt(fields[4]);
        // materialize bins to support equi-width/equi-height
        for (int i = 0; i < nbins; i++) {
            String lbound = String.valueOf(min + i * binwidth);
            String ubound = String.valueOf(min + (i + 1) * binwidth);
            ret.set(i, colID - 1, lbound + Lop.DATATYPE_PREFIX + ubound);
        }
        ret.getColumnMetadata(colID - 1).setNumDistinct((long) nbins);
    }
    // encode impute meta data into frame
    for (Entry<String, String> e : mvmeta.entrySet()) {
        int colID = ArrayUtils.indexOf(colnames, e.getKey()) + 1;
        String mvVal = e.getValue().split(TfUtils.TXMTD_SEP)[1];
        ret.getColumnMetadata(colID - 1).setMvValue(mvVal);
    }
    return ret;
}

Example 52 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class BooleanUnaryCPInstruction method parseInstruction.

public static BooleanUnaryCPInstruction parseInstruction(String str) throws DMLRuntimeException {
    CPOperand in = new CPOperand("", ValueType.UNKNOWN, DataType.UNKNOWN);
    CPOperand out = new CPOperand("", ValueType.UNKNOWN, DataType.UNKNOWN);
    String opcode = parseUnaryInstruction(str, in, out);
    // Boolean operations must be performed on BOOLEAN
    ValueType vt1 = in.getValueType();
    ValueType vt2 = out.getValueType();
    if (vt1 != ValueType.BOOLEAN || vt2 != ValueType.BOOLEAN)
        throw new DMLRuntimeException("Unexpected ValueType in ArithmeticInstruction.");
    // Determine appropriate Function Object based on opcode	
    return new BooleanUnaryCPInstruction(getSimpleUnaryOperator(opcode), in, out, opcode, str);
}

Example 53 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class BooleanBinaryCPInstruction method parseInstruction.

public static BooleanBinaryCPInstruction parseInstruction(String str) throws DMLRuntimeException {
    CPOperand in1 = new CPOperand("", ValueType.UNKNOWN, DataType.UNKNOWN);
    CPOperand in2 = new CPOperand("", ValueType.UNKNOWN, DataType.UNKNOWN);
    CPOperand out = new CPOperand("", ValueType.UNKNOWN, DataType.UNKNOWN);
    String opcode = parseBinaryInstruction(str, in1, in2, out);
    // Boolean operations must be performed on BOOLEAN
    ValueType vt1 = in1.getValueType();
    ValueType vt2 = in2.getValueType();
    ValueType vt3 = out.getValueType();
    if (vt1 != ValueType.BOOLEAN || vt3 != ValueType.BOOLEAN || (vt2 != null && vt2 != ValueType.BOOLEAN))
        throw new DMLRuntimeException("Unexpected ValueType in ArithmeticInstruction.");
    // Determine appropriate Function Object based on opcode	
    BinaryOperator bop = InstructionUtils.parseBinaryOperator(opcode);
    return new BooleanBinaryCPInstruction(bop, in1, in2, out, opcode, str);
}

Example 54 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class MLContextConversionUtil method dataFrameToFrameObject.

/**
	 * Convert a {@code DataFrame} to a {@code FrameObject}.
	 * 
	 * @param variableName
	 *            name of the variable associated with the frame
	 * @param dataFrame
	 *            the Spark {@code DataFrame}
	 * @param frameMetadata
	 *            the frame metadata
	 * @return the {@code DataFrame} frame converted to a converted to a
	 *         {@code FrameObject}
	 */
public static FrameObject dataFrameToFrameObject(String variableName, Dataset<Row> dataFrame, FrameMetadata frameMetadata) {
    try {
        //setup meta data and java spark context
        if (frameMetadata == null)
            frameMetadata = new FrameMetadata();
        determineFrameFormatIfNeeded(dataFrame, frameMetadata);
        boolean containsID = isDataFrameWithIDColumn(frameMetadata);
        MatrixCharacteristics mc = frameMetadata.asMatrixCharacteristics();
        if (mc == null)
            mc = new MatrixCharacteristics();
        //convert data frame and obtain column names / schema
        //TODO extend frame schema by column names (right now dropped)
        Pair<String[], ValueType[]> ret = new Pair<String[], ValueType[]>();
        JavaPairRDD<Long, FrameBlock> binaryBlock = FrameRDDConverterUtils.dataFrameToBinaryBlock(jsc(), dataFrame, mc, containsID, ret);
        frameMetadata.setFrameSchema(new FrameSchema(Arrays.asList(ret.getValue())));
        //required due to meta data copy
        frameMetadata.setMatrixCharacteristics(mc);
        return MLContextConversionUtil.binaryBlocksToFrameObject(variableName, binaryBlock, frameMetadata);
    } catch (DMLRuntimeException e) {
        throw new MLContextException("Exception converting DataFrame to FrameObject", e);
    }
}

Example 55 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class MLContextConversionUtil method javaRDDStringIJVToFrameObject.

/**
	 * Convert a {@code JavaRDD<String>} in IJV format to a {@code FrameObject}
	 * . Note that metadata is required for IJV format.
	 * 
	 * @param variableName
	 *            name of the variable associated with the frame
	 * @param javaRDD
	 *            the Java RDD of strings
	 * @param frameMetadata
	 *            frame metadata
	 * @return the {@code JavaRDD<String>} converted to a {@code FrameObject}
	 */
public static FrameObject javaRDDStringIJVToFrameObject(String variableName, JavaRDD<String> javaRDD, FrameMetadata frameMetadata) {
    JavaPairRDD<LongWritable, Text> javaPairRDD = javaRDD.mapToPair(new ConvertStringToLongTextPair());
    MatrixCharacteristics mc = (frameMetadata != null) ? frameMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
    JavaPairRDD<LongWritable, Text> javaPairRDDText = javaPairRDD.mapToPair(new CopyTextInputFunction());
    FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(), new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo), frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
    JavaPairRDD<Long, FrameBlock> rdd;
    try {
        ValueType[] lschema = null;
        if (lschema == null)
            lschema = UtilFunctions.nCopies((int) mc.getCols(), ValueType.STRING);
        rdd = FrameRDDConverterUtils.textCellToBinaryBlock(jsc(), javaPairRDDText, mc, lschema);
    } catch (DMLRuntimeException e) {
        e.printStackTrace();
        return null;
    }
    frameObject.setRDDHandle(new RDDObject(rdd, variableName));
    return frameObject;
}