Examples with OutputInfo org.apache.sysml.runtime.matrix.data.OutputInfo used on opensource projects

Example 11 with OutputInfo

use of org.apache.sysml.runtime.matrix.data.OutputInfo in project incubator-systemml by apache.

the class DynamicReadMatrixCP method execute.

@Override
public void execute() {
    try {
        String fname = ((Scalar) this.getFunctionInput(0)).getValue();
        Integer m = Integer.parseInt(((Scalar) this.getFunctionInput(1)).getValue());
        Integer n = Integer.parseInt(((Scalar) this.getFunctionInput(2)).getValue());
        String format = ((Scalar) this.getFunctionInput(3)).getValue();
        InputInfo ii = InputInfo.stringToInputInfo(format);
        OutputInfo oi = OutputInfo.BinaryBlockOutputInfo;
        MatrixBlock mbTmp = DataConverter.readMatrixFromHDFS(fname, ii, m, n, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
        String fnameTmp = createOutputFilePathAndName("TMP");
        _ret = new Matrix(fnameTmp, m, n, ValueType.Double);
        _ret.setMatrixDoubleArray(mbTmp, oi, ii);
    // NOTE: The packagesupport wrapper creates a new MatrixObjectNew with the given
    // matrix block. This leads to a dirty state of the new object. Hence, the resulting
    // intermediate plan variable will be exported in front of MR jobs and during this export
    // the format will be changed to binary block (the contract of external functions),
    // no matter in which format the original matrix was.
    } catch (Exception e) {
        throw new RuntimeException("Error executing dynamic read of matrix", e);
    }
}

Example 12 with OutputInfo

use of org.apache.sysml.runtime.matrix.data.OutputInfo in project incubator-systemml by apache.

the class DynamicReadMatrixRcCP method execute.

@Override
public void execute() {
    try {
        String fname = ((Scalar) this.getFunctionInput(0)).getValue();
        Integer m = Integer.parseInt(((Scalar) this.getFunctionInput(1)).getValue());
        Integer n = Integer.parseInt(((Scalar) this.getFunctionInput(2)).getValue());
        String format = ((Scalar) this.getFunctionInput(3)).getValue();
        InputInfo ii = InputInfo.stringToInputInfo(format);
        OutputInfo oi = OutputInfo.BinaryBlockOutputInfo;
        String fnameTmp = createOutputFilePathAndName("TMP");
        _ret = new Matrix(fnameTmp, m, n, ValueType.Double);
        MatrixBlock mbTmp = DataConverter.readMatrixFromHDFS(fname, ii, m, n, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
        _ret.setMatrixDoubleArray(mbTmp, oi, ii);
        _rc = new Scalar(ScalarValueType.Integer, "0");
    // NOTE: The packagesupport wrapper creates a new MatrixObjectNew with the given
    // matrix block. This leads to a dirty state of the new object. Hence, the resulting
    // intermediate plan variable will be exported in front of MR jobs and during this export
    // the format will be changed to binary block (the contract of external functions),
    // no matter in which format the original matrix was.
    } catch (Exception e) {
        _rc = new Scalar(ScalarValueType.Integer, "1");
    // throw new PackageRuntimeException("Error executing dynamic read of matrix",e);
    }
}

Example 13 with OutputInfo

use of org.apache.sysml.runtime.matrix.data.OutputInfo in project incubator-systemml by apache.

the class DynamicWriteMatrixCP method execute.

@Override
public void execute() {
    boolean success = false;
    try {
        Matrix mat = (Matrix) this.getFunctionInput(0);
        String fname = ((Scalar) this.getFunctionInput(1)).getValue();
        String format = ((Scalar) this.getFunctionInput(2)).getValue();
        MatrixObject mo = mat.getMatrixObject();
        MatrixCharacteristics mc = mo.getMatrixCharacteristics();
        OutputInfo oi = OutputInfo.stringToOutputInfo(format);
        MatrixBlock mb = mo.acquireRead();
        DataConverter.writeMatrixToHDFS(mb, fname, oi, mc);
        mo.release();
        success = true;
    } catch (Exception e) {
        throw new RuntimeException("Error executing dynamic write of matrix", e);
    }
    _success = new Scalar(ScalarValueType.Boolean, String.valueOf(success));
}

Example 14 with OutputInfo

use of org.apache.sysml.runtime.matrix.data.OutputInfo in project incubator-systemml by apache.

the class SortMR method runStitchupJob.

private static boolean runStitchupJob(String input, long rlen, long clen, int brlen, int bclen, long[] counts, int numReducers, int replication, String output) throws Exception {
    JobConf job = new JobConf(SortMR.class);
    job.setJobName("SortIndexesMR");
    // setup input/output paths
    Path inpath = new Path(input);
    Path outpath = new Path(output);
    FileInputFormat.setInputPaths(job, inpath);
    FileOutputFormat.setOutputPath(job, outpath);
    MapReduceTool.deleteFileIfExistOnHDFS(outpath, job);
    // set number of reducers (1 if local mode)
    if (InfrastructureAnalyzer.isLocalMode(job))
        job.setNumReduceTasks(1);
    else
        MRJobConfiguration.setNumReducers(job, numReducers, numReducers);
    // setup input/output format
    InputInfo iinfo = InputInfo.BinaryBlockInputInfo;
    OutputInfo oinfo = OutputInfo.BinaryBlockOutputInfo;
    job.setInputFormat(iinfo.inputFormatClass);
    job.setOutputFormat(oinfo.outputFormatClass);
    CompactInputFormat.setKeyValueClasses(job, MatrixIndexes.class, MatrixBlock.class);
    // setup mapper/reducer/output classes
    MRJobConfiguration.setInputInfo(job, (byte) 0, InputInfo.BinaryBlockInputInfo, brlen, bclen, ConvertTarget.BLOCK);
    job.setMapperClass(IndexSortStitchupMapper.class);
    job.setReducerClass(IndexSortStitchupReducer.class);
    job.setOutputKeyClass(oinfo.outputKeyClass);
    job.setOutputValueClass(oinfo.outputValueClass);
    MRJobConfiguration.setBlockSize(job, (byte) 0, brlen, bclen);
    MRJobConfiguration.setMatricesDimensions(job, new byte[] { 0 }, new long[] { rlen }, new long[] { clen });
    // compute shifted prefix sum of offsets and put into configuration
    long[] cumsumCounts = new long[counts.length];
    long sum = 0;
    for (int i = 0; i < counts.length; i++) {
        cumsumCounts[i] = sum;
        sum += counts[i];
    }
    job.set(SORT_INDEXES_OFFSETS, Arrays.toString(cumsumCounts));
    // setup replication factor
    job.setInt(MRConfigurationNames.DFS_REPLICATION, replication);
    // set unique working dir
    MRJobConfiguration.setUniqueWorkingDir(job);
    // run mr job
    RunningJob runJob = JobClient.runJob(job);
    return runJob.isSuccessful();
}

Example 15 with OutputInfo

use of org.apache.sysml.runtime.matrix.data.OutputInfo in project incubator-systemml by apache.

the class FrameConverterTest method runFrameConverterTest.

/**
 * @param schema
 * @param type
 * @param instType
 */
private void runFrameConverterTest(ValueType[] schema, ConvType type) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    DMLScript.rtplatform = RUNTIME_PLATFORM.SPARK;
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config);
        // data generation
        double[][] A = getRandomMatrix(rows, schema.length, -10, 10, 0.9, 2373);
        // prepare input/output infos
        OutputInfo oinfo = null;
        InputInfo iinfo = null;
        switch(type) {
            case CSV2BIN:
            case DFRM2BIN:
                oinfo = OutputInfo.CSVOutputInfo;
                iinfo = InputInfo.BinaryBlockInputInfo;
                break;
            case BIN2CSV:
                oinfo = OutputInfo.BinaryBlockOutputInfo;
                iinfo = InputInfo.CSVInputInfo;
                break;
            case TXTCELL2BIN:
                oinfo = OutputInfo.TextCellOutputInfo;
                iinfo = InputInfo.BinaryBlockInputInfo;
                break;
            case BIN2TXTCELL:
                oinfo = OutputInfo.BinaryBlockOutputInfo;
                iinfo = InputInfo.TextCellInputInfo;
                break;
            case MAT2BIN:
            case BIN2DFRM:
                oinfo = OutputInfo.BinaryBlockOutputInfo;
                iinfo = InputInfo.BinaryBlockInputInfo;
                break;
            case BIN2MAT:
                oinfo = OutputInfo.BinaryBlockOutputInfo;
                iinfo = InputInfo.BinaryBlockInputInfo;
                break;
            default:
                throw new RuntimeException("Unsuported converter type: " + type.toString());
        }
        if (type == ConvType.MAT2BIN || type == ConvType.BIN2MAT)
            runMatrixConverterAndVerify(schema, A, type, iinfo, oinfo);
        else
            runConverterAndVerify(schema, A, type, iinfo, oinfo);
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    } finally {
        DMLScript.rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}