Examples with ValueType org.apache.sysml.parser.Expression.ValueType used on opensource projects

Example 26 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class FrameSchemaReadTest method runFrameSchemaReadTest.

/**
 * @param testname
 * @param schema
 * @param wildcard
 */
private void runFrameSchemaReadTest(String testname, ValueType[] schema, boolean wildcard) {
    try {
        TestConfiguration config = getTestConfiguration(testname);
        loadTestConfiguration(config);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + testname + ".dml";
        programArgs = new String[] { "-explain", "-args", input("A"), getSchemaString(schema, wildcard), Integer.toString(rows), Integer.toString(schema.length), output("B") };
        // data generation
        double[][] A = getRandomMatrix(rows, schema.length, -10, 10, 0.9, 2373);
        // prepare input/output infos
        FrameBlock frame1 = new FrameBlock(schema);
        initFrameData(frame1, A, schema);
        // write frame data to hdfs
        FrameWriter writer = FrameWriterFactory.createFrameWriter(OutputInfo.CSVOutputInfo);
        writer.writeFrameToHDFS(frame1, input("A"), rows, schema.length);
        // run testcase
        runTest(true, false, null, -1);
        // read frame data from hdfs (not via readers to test physical schema)
        FrameReader reader = FrameReaderFactory.createFrameReader(InputInfo.BinaryBlockInputInfo);
        FrameBlock frame2 = ((FrameReaderBinaryBlock) reader).readFirstBlock(output("B"));
        // verify output schema
        ValueType[] schemaExpected = (testname.equals(TEST_NAME2) || wildcard) ? Collections.nCopies(schema.length, ValueType.STRING).toArray(new ValueType[0]) : schema;
        for (int i = 0; i < schemaExpected.length; i++) {
            Assert.assertEquals("Wrong result: " + frame2.getSchema()[i] + ".", schemaExpected[i], frame2.getSchema()[i]);
        }
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    }
}

Example 27 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class ValueTypeMatrixScalarBuiltinTest method runTest.

private void runTest(String testName, ValueType vtIn) {
    loadTestConfiguration(getTestConfiguration(testName));
    // setup arguments and run test
    String RI_HOME = SCRIPT_DIR + TEST_DIR;
    fullDMLScriptName = RI_HOME + testName + ".dml";
    programArgs = new String[] { "-args", vtIn == ValueType.DOUBLE ? "7.7" : "7", output("R") };
    runTest(true, false, null, -1);
    // check output value type
    ValueType vtOut = readDMLMetaDataValueType("R");
    Assert.assertTrue("Wrong output value type: " + vtOut.name(), vtOut.equals(ValueType.DOUBLE));
}

Example 28 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class MLContextFrameTest method testInputFrameAndMatrixOutputMatrix.

@Test
public void testInputFrameAndMatrixOutputMatrix() {
    System.out.println("MLContextFrameTest - input frame and matrix, output matrix");
    List<String> dataA = new ArrayList<String>();
    dataA.add("Test1,4.0");
    dataA.add("Test2,5.0");
    dataA.add("Test3,6.0");
    JavaRDD<String> javaRddStringA = sc.parallelize(dataA);
    ValueType[] schema = { ValueType.STRING, ValueType.DOUBLE };
    List<String> dataB = new ArrayList<String>();
    dataB.add("1.0");
    dataB.add("2.0");
    JavaRDD<String> javaRddStringB = sc.parallelize(dataB);
    JavaRDD<Row> javaRddRowA = FrameRDDConverterUtils.csvToRowRDD(sc, javaRddStringA, CSV_DELIM, schema);
    JavaRDD<Row> javaRddRowB = javaRddStringB.map(new CommaSeparatedValueStringToDoubleArrayRow());
    List<StructField> fieldsA = new ArrayList<StructField>();
    fieldsA.add(DataTypes.createStructField("1", DataTypes.StringType, true));
    fieldsA.add(DataTypes.createStructField("2", DataTypes.DoubleType, true));
    StructType schemaA = DataTypes.createStructType(fieldsA);
    Dataset<Row> dataFrameA = spark.createDataFrame(javaRddRowA, schemaA);
    List<StructField> fieldsB = new ArrayList<StructField>();
    fieldsB.add(DataTypes.createStructField("1", DataTypes.DoubleType, true));
    StructType schemaB = DataTypes.createStructType(fieldsB);
    Dataset<Row> dataFrameB = spark.createDataFrame(javaRddRowB, schemaB);
    String dmlString = "[tA, tAM] = transformencode (target = A, spec = \"{ids: true ,recode: [ 1, 2 ]}\");\n" + "C = tA %*% B;\n" + "M = s * C;";
    Script script = dml(dmlString).in("A", dataFrameA, new FrameMetadata(FrameFormat.CSV, dataFrameA.count(), (long) dataFrameA.columns().length)).in("B", dataFrameB, new MatrixMetadata(MatrixFormat.CSV, dataFrameB.count(), (long) dataFrameB.columns().length)).in("s", 2).out("M");
    MLResults results = ml.execute(script);
    double[][] matrix = results.getMatrixAs2DDoubleArray("M");
    Assert.assertEquals(6.0, matrix[0][0], 0.0);
    Assert.assertEquals(12.0, matrix[1][0], 0.0);
    Assert.assertEquals(18.0, matrix[2][0], 0.0);
}

Example 29 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class MLContextConversionUtil method javaRDDStringIJVToFrameObject.

/**
 * Convert a {@code JavaRDD<String>} in IJV format to a {@code FrameObject}
 * . Note that metadata is required for IJV format.
 *
 * @param javaRDD
 *            the Java RDD of strings
 * @param frameMetadata
 *            frame metadata
 * @return the {@code JavaRDD<String>} converted to a {@code FrameObject}
 */
public static FrameObject javaRDDStringIJVToFrameObject(JavaRDD<String> javaRDD, FrameMetadata frameMetadata) {
    JavaPairRDD<LongWritable, Text> javaPairRDD = javaRDD.mapToPair(new ConvertStringToLongTextPair());
    MatrixCharacteristics mc = (frameMetadata != null) ? frameMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
    JavaPairRDD<LongWritable, Text> javaPairRDDText = javaPairRDD.mapToPair(new CopyTextInputFunction());
    FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(), new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo), frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
    JavaPairRDD<Long, FrameBlock> rdd;
    try {
        ValueType[] lschema = null;
        if (lschema == null)
            lschema = UtilFunctions.nCopies((int) mc.getCols(), ValueType.STRING);
        rdd = FrameRDDConverterUtils.textCellToBinaryBlock(jsc(), javaPairRDDText, mc, lschema);
    } catch (DMLRuntimeException e) {
        e.printStackTrace();
        return null;
    }
    frameObject.setRDDHandle(new RDDObject(rdd));
    return frameObject;
}

Example 30 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class RewriteRemoveUnnecessaryCasts method rule_RemoveUnnecessaryCasts.

@SuppressWarnings("unchecked")
private void rule_RemoveUnnecessaryCasts(Hop hop) {
    // check mark processed
    if (hop.isVisited())
        return;
    // recursively process childs
    ArrayList<Hop> inputs = hop.getInput();
    for (int i = 0; i < inputs.size(); i++) rule_RemoveUnnecessaryCasts(inputs.get(i));
    // remove unnecessary value type cast
    if (hop instanceof UnaryOp && HopRewriteUtils.isValueTypeCast(((UnaryOp) hop).getOp())) {
        Hop in = hop.getInput().get(0);
        // type cast input
        ValueType vtIn = in.getValueType();
        // type cast output
        ValueType vtOut = hop.getValueType();
        // if input/output types match, no need to cast
        if (vtIn == vtOut && vtIn != ValueType.UNKNOWN) {
            ArrayList<Hop> parents = hop.getParent();
            for (// for all parents
            int i = 0; // for all parents
            i < parents.size(); // for all parents
            i++) {
                Hop p = parents.get(i);
                ArrayList<Hop> pin = p.getInput();
                for (// for all parent childs
                int j = 0; // for all parent childs
                j < pin.size(); // for all parent childs
                j++) {
                    Hop pinj = pin.get(j);
                    if (// found parent ref
                    pinj == hop) {
                        // rehang cast input as child of cast consumer
                        // remove cast ref
                        pin.remove(j);
                        // add ref to cast input
                        pin.add(j, in);
                        // remove cast from cast input parents
                        in.getParent().remove(hop);
                        // add parent to cast input parents
                        in.getParent().add(p);
                    }
                }
            }
            parents.clear();
        }
    }
    // remove unnecessary data type casts
    if (hop instanceof UnaryOp && hop.getInput().get(0) instanceof UnaryOp) {
        UnaryOp uop1 = (UnaryOp) hop;
        UnaryOp uop2 = (UnaryOp) hop.getInput().get(0);
        if ((uop1.getOp() == OpOp1.CAST_AS_MATRIX && uop2.getOp() == OpOp1.CAST_AS_SCALAR) || (uop1.getOp() == OpOp1.CAST_AS_SCALAR && uop2.getOp() == OpOp1.CAST_AS_MATRIX)) {
            Hop input = uop2.getInput().get(0);
            // rewire parents
            ArrayList<Hop> parents = (ArrayList<Hop>) hop.getParent().clone();
            for (Hop p : parents) HopRewriteUtils.replaceChildReference(p, hop, input);
        }
    }
    // mark processed
    hop.setVisited();
}