use of org.apache.sysml.lops.CSVReBlock in project incubator-systemml by apache.
the class Hop method constructAndSetReblockLopIfRequired.
private void constructAndSetReblockLopIfRequired() {
// determine execution type
ExecType et = ExecType.CP;
if (DMLScript.rtplatform != RUNTIME_PLATFORM.SINGLE_NODE && !(getDataType() == DataType.SCALAR)) {
et = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
}
// add reblock lop to output if required
if (_requiresReblock && et != ExecType.CP) {
Lop input = getLops();
Lop reblock = null;
try {
if (// CSV
this instanceof DataOp && ((DataOp) this).getDataOpType() == DataOpTypes.PERSISTENTREAD && ((DataOp) this).getInputFormatType() == FileFormatTypes.CSV) {
reblock = new CSVReBlock(input, getRowsInBlock(), getColsInBlock(), getDataType(), getValueType(), et);
} else // TEXT / MM / BINARYBLOCK / BINARYCELL
{
reblock = new ReBlock(input, getRowsInBlock(), getColsInBlock(), getDataType(), getValueType(), _outputEmptyBlocks, et);
}
} catch (LopsException ex) {
throw new HopsException(ex);
}
setOutputDimensions(reblock);
setLineNumbers(reblock);
setLops(reblock);
}
}
use of org.apache.sysml.lops.CSVReBlock in project systemml by apache.
the class Hop method constructAndSetReblockLopIfRequired.
private void constructAndSetReblockLopIfRequired() {
// determine execution type
ExecType et = ExecType.CP;
if (DMLScript.rtplatform != RUNTIME_PLATFORM.SINGLE_NODE && !(getDataType() == DataType.SCALAR)) {
et = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
}
// add reblock lop to output if required
if (_requiresReblock && et != ExecType.CP) {
Lop input = getLops();
Lop reblock = null;
try {
if (// CSV
this instanceof DataOp && ((DataOp) this).getDataOpType() == DataOpTypes.PERSISTENTREAD && ((DataOp) this).getInputFormatType() == FileFormatTypes.CSV) {
reblock = new CSVReBlock(input, getRowsInBlock(), getColsInBlock(), getDataType(), getValueType(), et);
} else // TEXT / MM / BINARYBLOCK / BINARYCELL
{
reblock = new ReBlock(input, getRowsInBlock(), getColsInBlock(), getDataType(), getValueType(), _outputEmptyBlocks, et);
}
} catch (LopsException ex) {
throw new HopsException(ex);
}
setOutputDimensions(reblock);
setLineNumbers(reblock);
setLops(reblock);
}
}
Aggregations