Examples with CSVReBlock org.apache.sysml.lops.CSVReBlock used on opensource projects

Example 1 with CSVReBlock

use of org.apache.sysml.lops.CSVReBlock in project incubator-systemml by apache.

the class Hop method constructAndSetReblockLopIfRequired.

private void constructAndSetReblockLopIfRequired() {
    // determine execution type
    ExecType et = ExecType.CP;
    if (DMLScript.rtplatform != RUNTIME_PLATFORM.SINGLE_NODE && !(getDataType() == DataType.SCALAR)) {
        et = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
    }
    // add reblock lop to output if required
    if (_requiresReblock && et != ExecType.CP) {
        Lop input = getLops();
        Lop reblock = null;
        try {
            if (// CSV
            this instanceof DataOp && ((DataOp) this).getDataOpType() == DataOpTypes.PERSISTENTREAD && ((DataOp) this).getInputFormatType() == FileFormatTypes.CSV) {
                reblock = new CSVReBlock(input, getRowsInBlock(), getColsInBlock(), getDataType(), getValueType(), et);
            } else // TEXT / MM / BINARYBLOCK / BINARYCELL
            {
                reblock = new ReBlock(input, getRowsInBlock(), getColsInBlock(), getDataType(), getValueType(), _outputEmptyBlocks, et);
            }
        } catch (LopsException ex) {
            throw new HopsException(ex);
        }
        setOutputDimensions(reblock);
        setLineNumbers(reblock);
        setLops(reblock);
    }
}

Example 2 with CSVReBlock

use of org.apache.sysml.lops.CSVReBlock in project systemml by apache.

the class Hop method constructAndSetReblockLopIfRequired.

private void constructAndSetReblockLopIfRequired() {
    // determine execution type
    ExecType et = ExecType.CP;
    if (DMLScript.rtplatform != RUNTIME_PLATFORM.SINGLE_NODE && !(getDataType() == DataType.SCALAR)) {
        et = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
    }
    // add reblock lop to output if required
    if (_requiresReblock && et != ExecType.CP) {
        Lop input = getLops();
        Lop reblock = null;
        try {
            if (// CSV
            this instanceof DataOp && ((DataOp) this).getDataOpType() == DataOpTypes.PERSISTENTREAD && ((DataOp) this).getInputFormatType() == FileFormatTypes.CSV) {
                reblock = new CSVReBlock(input, getRowsInBlock(), getColsInBlock(), getDataType(), getValueType(), et);
            } else // TEXT / MM / BINARYBLOCK / BINARYCELL
            {
                reblock = new ReBlock(input, getRowsInBlock(), getColsInBlock(), getDataType(), getValueType(), _outputEmptyBlocks, et);
            }
        } catch (LopsException ex) {
            throw new HopsException(ex);
        }
        setOutputDimensions(reblock);
        setLineNumbers(reblock);
        setLops(reblock);
    }
}