Search in sources :

Example 21 with FrameReader

use of org.apache.sysml.runtime.io.FrameReader in project incubator-systemml by apache.

the class TransformReadMetaTest method runTransformReadMetaTest.

/**
	 * 
	 * @param sparseM1
	 * @param sparseM2
	 * @param instType
	 * @throws IOException 
	 * @throws DMLRuntimeException 
	 */
private void runTransformReadMetaTest(RUNTIME_PLATFORM rt, String ofmt, String delim) throws IOException, DMLRuntimeException {
    RUNTIME_PLATFORM platformOld = rtplatform;
    rtplatform = rt;
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        String testname = delim.equals(",") ? TEST_NAME1 : TEST_NAME2;
        getAndLoadTestConfiguration(testname);
        //generate input data
        double[][] X = DataConverter.convertToDoubleMatrix(MatrixBlock.seqOperations(0.5, rows / 2, 0.5).appendOperations(MatrixBlock.seqOperations(0.5, rows / 2, 0.5), new MatrixBlock()));
        MatrixBlock mbX = DataConverter.convertToMatrixBlock(X);
        CSVFileFormatProperties fprops = new CSVFileFormatProperties(false, delim, false);
        MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(OutputInfo.CSVOutputInfo, 1, fprops);
        writer.writeMatrixToHDFS(mbX, input("X"), rows, 2, -1, -1, -1);
        //read specs transform X and Y
        String specX = MapReduceTool.readStringFromHDFSFile(SCRIPT_DIR + TEST_DIR + SPEC_X);
        fullDMLScriptName = SCRIPT_DIR + TEST_DIR + testname + ".dml";
        programArgs = new String[] { "-args", input("X"), specX, output("M1"), output("M"), ofmt, delim };
        //run test
        runTest(true, false, null, -1);
        //compare meta data frames
        InputInfo iinfo = InputInfo.stringExternalToInputInfo(ofmt);
        FrameReader reader = FrameReaderFactory.createFrameReader(iinfo);
        FrameBlock mExpected = TfMetaUtils.readTransformMetaDataFromFile(specX, output("M1"), delim);
        FrameBlock mRet = reader.readFrameFromHDFS(output("M"), rows, 2);
        for (int i = 0; i < rows; i++) for (int j = 0; j < 2; j++) {
            Assert.assertTrue("Wrong result: " + mRet.get(i, j) + ".", UtilFunctions.compareTo(ValueType.STRING, mExpected.get(i, j), mRet.get(i, j)) == 0);
        }
    } catch (Exception ex) {
        throw new IOException(ex);
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}
Also used : MatrixBlock(org.apache.sysml.runtime.matrix.data.MatrixBlock) CSVFileFormatProperties(org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties) IOException(java.io.IOException) DMLRuntimeException(org.apache.sysml.runtime.DMLRuntimeException) IOException(java.io.IOException) RUNTIME_PLATFORM(org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM) InputInfo(org.apache.sysml.runtime.matrix.data.InputInfo) FrameBlock(org.apache.sysml.runtime.matrix.data.FrameBlock) FrameReader(org.apache.sysml.runtime.io.FrameReader) MatrixWriter(org.apache.sysml.runtime.io.MatrixWriter)

Aggregations

FrameReader (org.apache.sysml.runtime.io.FrameReader)21 FrameBlock (org.apache.sysml.runtime.matrix.data.FrameBlock)18 RUNTIME_PLATFORM (org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM)8 CSVFileFormatProperties (org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties)8 IOException (java.io.IOException)7 DMLRuntimeException (org.apache.sysml.runtime.DMLRuntimeException)5 FrameWriter (org.apache.sysml.runtime.io.FrameWriter)5 MatrixCharacteristics (org.apache.sysml.runtime.matrix.MatrixCharacteristics)5 MatrixBlock (org.apache.sysml.runtime.matrix.data.MatrixBlock)5 InputInfo (org.apache.sysml.runtime.matrix.data.InputInfo)3 ValueType (org.apache.sysml.parser.Expression.ValueType)2 MatrixReader (org.apache.sysml.runtime.io.MatrixReader)2 MatrixWriter (org.apache.sysml.runtime.io.MatrixWriter)2 JavaPairRDD (org.apache.spark.api.java.JavaPairRDD)1 DMLException (org.apache.sysml.api.DMLException)1 LanguageException (org.apache.sysml.parser.LanguageException)1 ParseException (org.apache.sysml.parser.ParseException)1 FrameObject (org.apache.sysml.runtime.controlprogram.caching.FrameObject)1 SparkExecutionContext (org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext)1 RDDTransformApplyFunction (org.apache.sysml.runtime.instructions.spark.ParameterizedBuiltinSPInstruction.RDDTransformApplyFunction)1