Examples with ReaderBinaryBlock org.apache.sysml.runtime.io.ReaderBinaryBlock used on opensource projects

Example 1 with ReaderBinaryBlock

use of org.apache.sysml.runtime.io.ReaderBinaryBlock in project incubator-systemml by apache.

the class DistributedCacheInput method readDataBlocks.

private void readDataBlocks(int rowBlockIndex, int colBlockIndex) throws DMLRuntimeException {
    //get filename for rowblock/colblock
    String fname = _localFilePath.toString();
    if (isPartitioned())
        fname = getPartitionFileName(rowBlockIndex, colBlockIndex);
    //read matrix partition (or entire vector)
    try {
        ReaderBinaryBlock reader = (ReaderBinaryBlock) MatrixReaderFactory.createMatrixReader(InputInfo.BinaryBlockInputInfo);
        reader.setLocalFS(!MRBaseForCommonInstructions.isJobLocal);
        ArrayList<IndexedMatrixValue> tmp = reader.readIndexedMatrixBlocksFromHDFS(fname, _rlen, _clen, _brlen, _bclen);
        int rowBlocks = (int) Math.ceil(_rlen / (double) _brlen);
        int colBlocks = (int) Math.ceil(_clen / (double) _bclen);
        if (dataBlocks == null)
            dataBlocks = new IndexedMatrixValue[rowBlocks][colBlocks];
        for (IndexedMatrixValue val : tmp) {
            MatrixIndexes idx = val.getIndexes();
            dataBlocks[(int) idx.getRowIndex() - 1][(int) idx.getColumnIndex() - 1] = val;
        }
    } catch (Exception ex) {
        throw new DMLRuntimeException(ex);
    }
}

Example 2 with ReaderBinaryBlock

use of org.apache.sysml.runtime.io.ReaderBinaryBlock in project incubator-systemml by apache.

the class ScalingTest method runScalingTest.

/**
	 * 
	 * @param sparseM1
	 * @param sparseM2
	 * @param instType
	 * @throws IOException 
	 * @throws DMLRuntimeException 
	 */
private void runScalingTest(int rows, int cols, RUNTIME_PLATFORM rt, String ofmt) throws IOException, DMLRuntimeException, Exception {
    RUNTIME_PLATFORM platformOld = rtplatform;
    rtplatform = rt;
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config);
        String HOME = SCRIPT_DIR + TEST_DIR;
        String specFile = input("spec.json");
        String inputFile = input("X");
        String outputFile = output(config.getOutputFiles()[0]);
        String outputFileR = expected(config.getOutputFiles()[0]);
        generateSpecFile(cols, specFile);
        // This is for running the junit test the new way, i.e., construct the arguments directly
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-nvargs", "DATA=" + inputFile, "TFSPEC=" + specFile, "TFMTD=" + output("tfmtd"), "TFDATA=" + outputFile, "OFMT=" + ofmt };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputFile + " " + outputFileR;
        //generate actual dataset 
        double[][] X = getRandomMatrix(rows, cols, -50, 50, 1.0, 7);
        TestUtils.writeCSVTestMatrix(inputFile, X);
        generateFrameMTD(inputFile);
        runTest(true, false, null, -1);
        runRScript(true);
        ReaderTextCSV expReader = new ReaderTextCSV(new CSVFileFormatProperties(false, ",", true, 0, null));
        MatrixBlock exp = expReader.readMatrixFromHDFS(outputFileR, -1, -1, -1, -1, -1);
        MatrixBlock out = null;
        if (ofmt.equals("csv")) {
            ReaderTextCSV outReader = new ReaderTextCSV(new CSVFileFormatProperties(false, ",", true, 0, null));
            out = outReader.readMatrixFromHDFS(outputFile, -1, -1, -1, -1, -1);
        } else {
            ReaderBinaryBlock bbReader = new ReaderBinaryBlock(false);
            out = bbReader.readMatrixFromHDFS(outputFile, exp.getNumRows(), exp.getNumColumns(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), -1);
        }
        assertTrue("Incorrect output from data transform.", TransformTest.equals(out, exp, 1e-10));
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 3 with ReaderBinaryBlock

use of org.apache.sysml.runtime.io.ReaderBinaryBlock in project incubator-systemml by apache.

the class TransformTest method runTransformTest.

// ------------------------------
private void runTransformTest(RUNTIME_PLATFORM rt, String ofmt, String dataset, boolean byid) {
    String DATASET = null, SPEC = null, TFDATA = null;
    if (dataset.equals("homes")) {
        DATASET = HOMES_DATASET;
        SPEC = (byid ? HOMES_IDSPEC : HOMES_SPEC);
        TFDATA = HOMES_TFDATA;
    } else if (dataset.equals("homesomit")) {
        DATASET = HOMES_OMIT_DATASET;
        SPEC = (byid ? HOMES_OMIT_IDSPEC : HOMES_OMIT_SPEC);
        TFDATA = HOMES_OMIT_TFDATA;
    } else if (dataset.equals("homes2")) {
        DATASET = HOMES2_DATASET;
        SPEC = (byid ? HOMES2_IDSPEC : HOMES2_SPEC);
        TFDATA = HOMES2_TFDATA;
    } else if (dataset.equals("iris")) {
        DATASET = IRIS_DATASET;
        SPEC = (byid ? IRIS_IDSPEC : IRIS_SPEC);
        TFDATA = IRIS_TFDATA;
    }
    RUNTIME_PLATFORM rtold = rtplatform;
    rtplatform = rt;
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        getAndLoadTestConfiguration(TEST_NAME1);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME1 + ".dml";
        programArgs = new String[] { "-nvargs", "DATA=" + HOME + "input/" + DATASET, "TFSPEC=" + HOME + "input/" + SPEC, "TFMTD=" + output("tfmtd"), "TFDATA=" + output("tfout"), "OFMT=" + ofmt };
        boolean exceptionExpected = false;
        runTest(true, exceptionExpected, null, -1);
        fullDMLScriptName = HOME + TEST_NAME2 + ".dml";
        programArgs = new String[] { "-nvargs", "DATA=" + HOME + "input/" + DATASET, // generated above
        "APPLYMTD=" + output("tfmtd"), "TFMTD=" + output("test_tfmtd"), "TFDATA=" + output("test_tfout"), "OFMT=" + ofmt };
        exceptionExpected = false;
        runTest(true, exceptionExpected, null, -1);
        try {
            ReaderTextCSV csvReader = new ReaderTextCSV(new CSVFileFormatProperties(true, ",", true, 0, null));
            MatrixBlock exp = csvReader.readMatrixFromHDFS(HOME + "input/" + TFDATA, -1, -1, -1, -1, -1);
            MatrixBlock out = null, out2 = null;
            if (ofmt.equals("csv")) {
                ReaderTextCSV outReader = new ReaderTextCSV(new CSVFileFormatProperties(false, ",", true, 0, null));
                out = outReader.readMatrixFromHDFS(output("tfout"), -1, -1, -1, -1, -1);
                out2 = outReader.readMatrixFromHDFS(output("test_tfout"), -1, -1, -1, -1, -1);
            } else {
                ReaderBinaryBlock bbReader = new ReaderBinaryBlock(false);
                out = bbReader.readMatrixFromHDFS(output("tfout"), exp.getNumRows(), exp.getNumColumns(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), -1);
                out2 = bbReader.readMatrixFromHDFS(output("test_tfout"), exp.getNumRows(), exp.getNumColumns(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), -1);
            }
            assertTrue("Incorrect output from data transform.", equals(out, exp, 1e-10));
            assertTrue("Incorrect output from apply transform.", equals(out2, exp, 1e-10));
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    } finally {
        rtplatform = rtold;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}