Search in sources :

Example 1 with FrameReader

use of org.apache.sysml.runtime.io.FrameReader in project incubator-systemml by apache.

the class MultiReturnParameterizedBuiltinSPInstruction method processInstruction.

@Override
@SuppressWarnings("unchecked")
public void processInstruction(ExecutionContext ec) throws DMLRuntimeException {
    SparkExecutionContext sec = (SparkExecutionContext) ec;
    try {
        //get input RDD and meta data
        FrameObject fo = sec.getFrameObject(input1.getName());
        FrameObject fometa = sec.getFrameObject(_outputs.get(1).getName());
        JavaPairRDD<Long, FrameBlock> in = (JavaPairRDD<Long, FrameBlock>) sec.getRDDHandleForFrameObject(fo, InputInfo.BinaryBlockInputInfo);
        String spec = ec.getScalarInput(input2.getName(), input2.getValueType(), input2.isLiteral()).getStringValue();
        MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(input1.getName());
        MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName());
        String[] colnames = !TfMetaUtils.isIDSpecification(spec) ? in.lookup(1L).get(0).getColumnNames() : null;
        //step 1: build transform meta data
        Encoder encoderBuild = EncoderFactory.createEncoder(spec, colnames, fo.getSchema(), (int) fo.getNumColumns(), null);
        MaxLongAccumulator accMax = registerMaxLongAccumulator(sec.getSparkContext());
        JavaRDD<String> rcMaps = in.mapPartitionsToPair(new TransformEncodeBuildFunction(encoderBuild)).distinct().groupByKey().flatMap(new TransformEncodeGroupFunction(accMax));
        if (containsMVImputeEncoder(encoderBuild)) {
            MVImputeAgent mva = getMVImputeEncoder(encoderBuild);
            rcMaps = rcMaps.union(in.mapPartitionsToPair(new TransformEncodeBuild2Function(mva)).groupByKey().flatMap(new TransformEncodeGroup2Function(mva)));
        }
        //trigger eval
        rcMaps.saveAsTextFile(fometa.getFileName());
        //consolidate meta data frame (reuse multi-threaded reader, special handling missing values) 
        FrameReader reader = FrameReaderFactory.createFrameReader(InputInfo.TextCellInputInfo);
        FrameBlock meta = reader.readFrameFromHDFS(fometa.getFileName(), accMax.value(), fo.getNumColumns());
        //recompute num distinct items per column
        meta.recomputeColumnCardinality();
        meta.setColumnNames((colnames != null) ? colnames : meta.getColumnNames());
        //step 2: transform apply (similar to spark transformapply)
        //compute omit offset map for block shifts
        TfOffsetMap omap = null;
        if (TfMetaUtils.containsOmitSpec(spec, colnames)) {
            omap = new TfOffsetMap(SparkUtils.toIndexedLong(in.mapToPair(new RDDTransformApplyOffsetFunction(spec, colnames)).collect()));
        }
        //create encoder broadcast (avoiding replication per task) 
        Encoder encoder = EncoderFactory.createEncoder(spec, colnames, fo.getSchema(), (int) fo.getNumColumns(), meta);
        mcOut.setDimension(mcIn.getRows() - ((omap != null) ? omap.getNumRmRows() : 0), encoder.getNumCols());
        Broadcast<Encoder> bmeta = sec.getSparkContext().broadcast(encoder);
        Broadcast<TfOffsetMap> bomap = (omap != null) ? sec.getSparkContext().broadcast(omap) : null;
        //execute transform apply
        JavaPairRDD<Long, FrameBlock> tmp = in.mapToPair(new RDDTransformApplyFunction(bmeta, bomap));
        JavaPairRDD<MatrixIndexes, MatrixBlock> out = FrameRDDConverterUtils.binaryBlockToMatrixBlock(tmp, mcOut, mcOut);
        //set output and maintain lineage/output characteristics
        sec.setRDDHandleForVariable(_outputs.get(0).getName(), out);
        sec.addLineageRDD(_outputs.get(0).getName(), input1.getName());
        sec.setFrameOutput(_outputs.get(1).getName(), meta);
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
}
Also used : MatrixBlock(org.apache.sysml.runtime.matrix.data.MatrixBlock) DMLRuntimeException(org.apache.sysml.runtime.DMLRuntimeException) FrameBlock(org.apache.sysml.runtime.matrix.data.FrameBlock) Encoder(org.apache.sysml.runtime.transform.encode.Encoder) JavaPairRDD(org.apache.spark.api.java.JavaPairRDD) SparkExecutionContext(org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext) RDDTransformApplyOffsetFunction(org.apache.sysml.runtime.instructions.spark.ParameterizedBuiltinSPInstruction.RDDTransformApplyOffsetFunction) MatrixIndexes(org.apache.sysml.runtime.matrix.data.MatrixIndexes) FrameObject(org.apache.sysml.runtime.controlprogram.caching.FrameObject) IOException(java.io.IOException) MatrixCharacteristics(org.apache.sysml.runtime.matrix.MatrixCharacteristics) RDDTransformApplyFunction(org.apache.sysml.runtime.instructions.spark.ParameterizedBuiltinSPInstruction.RDDTransformApplyFunction) TfOffsetMap(org.apache.sysml.runtime.transform.meta.TfOffsetMap) FrameReader(org.apache.sysml.runtime.io.FrameReader) MVImputeAgent(org.apache.sysml.runtime.transform.MVImputeAgent)

Example 2 with FrameReader

use of org.apache.sysml.runtime.io.FrameReader in project incubator-systemml by apache.

the class Connection method readStringFrame.

/**
	 * Reads an input frame in arbitrary format from HDFS into a dense string array.
	 * NOTE: this call currently only supports default configurations for CSV.
	 * 
	 * @param fname the filename of the input frame
	 * @param iinfo InputInfo object
	 * @param rows number of rows in the frame
	 * @param cols number of columns in the frame
	 * @return frame as a two-dimensional string array
	 * @throws IOException if IOException occurs
	 */
public String[][] readStringFrame(String fname, InputInfo iinfo, long rows, long cols) throws IOException {
    try {
        FrameReader reader = FrameReaderFactory.createFrameReader(iinfo);
        FrameBlock mb = reader.readFrameFromHDFS(fname, rows, cols);
        return DataConverter.convertToStringFrame(mb);
    } catch (Exception ex) {
        throw new IOException(ex);
    }
}
Also used : FrameBlock(org.apache.sysml.runtime.matrix.data.FrameBlock) FrameReader(org.apache.sysml.runtime.io.FrameReader) IOException(java.io.IOException) DMLRuntimeException(org.apache.sysml.runtime.DMLRuntimeException) DMLException(org.apache.sysml.api.DMLException) IOException(java.io.IOException) ParseException(org.apache.sysml.parser.ParseException)

Example 3 with FrameReader

use of org.apache.sysml.runtime.io.FrameReader in project incubator-systemml by apache.

the class AutomatedTestBase method readDMLFrameFromHDFS.

protected static FrameBlock readDMLFrameFromHDFS(String fileName, InputInfo iinfo, MatrixCharacteristics md) throws DMLRuntimeException, IOException {
    //read frame data from hdfs
    String strFrameFileName = baseDirectory + OUTPUT_DIR + fileName;
    FrameReader reader = FrameReaderFactory.createFrameReader(iinfo);
    return reader.readFrameFromHDFS(strFrameFileName, md.getRows(), md.getCols());
}
Also used : FrameReader(org.apache.sysml.runtime.io.FrameReader)

Example 4 with FrameReader

use of org.apache.sysml.runtime.io.FrameReader in project incubator-systemml by apache.

the class AutomatedTestBase method readRFrameFromHDFS.

protected static FrameBlock readRFrameFromHDFS(String fileName, InputInfo iinfo, MatrixCharacteristics md) throws DMLRuntimeException, IOException {
    //read frame data from hdfs
    String strFrameFileName = baseDirectory + EXPECTED_DIR + fileName;
    CSVFileFormatProperties fprop = new CSVFileFormatProperties();
    fprop.setHeader(true);
    FrameReader reader = FrameReaderFactory.createFrameReader(iinfo, fprop);
    return reader.readFrameFromHDFS(strFrameFileName, md.getRows(), md.getCols());
}
Also used : CSVFileFormatProperties(org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties) FrameReader(org.apache.sysml.runtime.io.FrameReader)

Example 5 with FrameReader

use of org.apache.sysml.runtime.io.FrameReader in project incubator-systemml by apache.

the class FrameMatrixCastingTest method readMatrixOrFrameInput.

/**
	 * 
	 * @param fname
	 * @param rows
	 * @param cols
	 * @param dt
	 * @return
	 * @throws DMLRuntimeException
	 * @throws IOException
	 */
private double[][] readMatrixOrFrameInput(String fname, int rows, int cols, DataType dt) throws DMLRuntimeException, IOException {
    MatrixBlock ret = null;
    //read input data
    if (dt == DataType.FRAME) {
        FrameReader reader = FrameReaderFactory.createFrameReader(InputInfo.BinaryBlockInputInfo);
        FrameBlock fb = reader.readFrameFromHDFS(fname, rows, cols);
        ret = DataConverter.convertToMatrixBlock(fb);
    } else {
        int blksize = ConfigurationManager.getBlocksize();
        MatrixReader reader = MatrixReaderFactory.createMatrixReader(InputInfo.BinaryBlockInputInfo);
        ret = reader.readMatrixFromHDFS(fname, rows, cols, blksize, blksize, -1);
    }
    return DataConverter.convertToDoubleMatrix(ret);
}
Also used : MatrixBlock(org.apache.sysml.runtime.matrix.data.MatrixBlock) FrameBlock(org.apache.sysml.runtime.matrix.data.FrameBlock) FrameReader(org.apache.sysml.runtime.io.FrameReader) MatrixReader(org.apache.sysml.runtime.io.MatrixReader)

Aggregations

FrameReader (org.apache.sysml.runtime.io.FrameReader)20 FrameBlock (org.apache.sysml.runtime.matrix.data.FrameBlock)17 IOException (java.io.IOException)8 DMLRuntimeException (org.apache.sysml.runtime.DMLRuntimeException)8 RUNTIME_PLATFORM (org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM)7 CSVFileFormatProperties (org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties)7 FrameWriter (org.apache.sysml.runtime.io.FrameWriter)5 MatrixCharacteristics (org.apache.sysml.runtime.matrix.MatrixCharacteristics)5 MatrixBlock (org.apache.sysml.runtime.matrix.data.MatrixBlock)5 InputInfo (org.apache.sysml.runtime.matrix.data.InputInfo)3 ValueType (org.apache.sysml.parser.Expression.ValueType)2 MatrixReader (org.apache.sysml.runtime.io.MatrixReader)2 MatrixWriter (org.apache.sysml.runtime.io.MatrixWriter)2 JavaPairRDD (org.apache.spark.api.java.JavaPairRDD)1 DMLException (org.apache.sysml.api.DMLException)1 ParseException (org.apache.sysml.parser.ParseException)1 FrameObject (org.apache.sysml.runtime.controlprogram.caching.FrameObject)1 SparkExecutionContext (org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext)1 RDDTransformApplyFunction (org.apache.sysml.runtime.instructions.spark.ParameterizedBuiltinSPInstruction.RDDTransformApplyFunction)1 RDDTransformApplyOffsetFunction (org.apache.sysml.runtime.instructions.spark.ParameterizedBuiltinSPInstruction.RDDTransformApplyOffsetFunction)1