Examples with ValueType org.apache.sysml.parser.Expression.ValueType used on opensource projects

Example 6 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class FrameReaderTextCell method readFrameFromInputStream.

@Override
public final FrameBlock readFrameFromInputStream(InputStream is, ValueType[] schema, String[] names, long rlen, long clen) throws IOException, DMLRuntimeException {
    //allocate output frame block
    ValueType[] lschema = createOutputSchema(schema, clen);
    String[] lnames = createOutputNames(names, clen);
    FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);
    //core read 
    readRawTextCellFrameFromInputStream(is, ret, lschema, lnames, rlen, clen);
    return ret;
}

Example 7 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class FrameReaderTextCell method readFrameFromHDFS.

@Override
public final FrameBlock readFrameFromHDFS(String fname, ValueType[] schema, String[] names, long rlen, long clen) throws IOException, DMLRuntimeException {
    //allocate output frame block
    ValueType[] lschema = createOutputSchema(schema, clen);
    String[] lnames = createOutputNames(names, clen);
    FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);
    //prepare file access
    JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
    Path path = new Path(fname);
    FileSystem fs = IOUtilFunctions.getFileSystem(path, job);
    //check existence and non-empty file
    checkValidInputFile(fs, path);
    //core read (sequential/parallel)
    readTextCellFrameFromHDFS(path, job, fs, ret, lschema, lnames, rlen, clen);
    return ret;
}

Example 8 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class FrameReaderTextCell method readTextCellFrameFromInputSplit.

protected final void readTextCellFrameFromInputSplit(InputSplit split, TextInputFormat informat, JobConf job, FrameBlock dest) throws IOException {
    ValueType[] schema = dest.getSchema();
    int rlen = dest.getNumRows();
    int clen = dest.getNumColumns();
    //create record reader
    RecordReader<LongWritable, Text> reader = informat.getRecordReader(split, job, Reporter.NULL);
    LongWritable key = new LongWritable();
    Text value = new Text();
    FastStringTokenizer st = new FastStringTokenizer(' ');
    int row = -1;
    int col = -1;
    try {
        while (reader.next(key, value)) {
            //reinit tokenizer
            st.reset(value.toString());
            row = st.nextInt() - 1;
            col = st.nextInt() - 1;
            if (row == -3)
                dest.getColumnMetadata(col).setMvValue(st.nextToken());
            else if (row == -2)
                dest.getColumnMetadata(col).setNumDistinct(st.nextLong());
            else
                dest.set(row, col, UtilFunctions.stringToObject(schema[col], st.nextToken()));
        }
    } catch (Exception ex) {
        //post-mortem error handling and bounds checking
        if (row < 0 || row + 1 > rlen || col < 0 || col + 1 > clen) {
            throw new IOException("Frame cell [" + (row + 1) + "," + (col + 1) + "] " + "out of overall frame range [1:" + rlen + ",1:" + clen + "].");
        } else {
            throw new IOException("Unable to read frame in text cell format.", ex);
        }
    } finally {
        IOUtilFunctions.closeSilently(reader);
    }
}

Example 9 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class BinaryOp method constructLopsAppend.

private void constructLopsAppend(ExecType et) throws HopsException, LopsException {
    DataType dt1 = getInput().get(0).getDataType();
    DataType dt2 = getInput().get(1).getDataType();
    ValueType vt1 = getInput().get(0).getValueType();
    ValueType vt2 = getInput().get(1).getValueType();
    boolean cbind = op == OpOp2.CBIND;
    //sanity check for input data types
    if (!((dt1 == DataType.MATRIX && dt2 == DataType.MATRIX) || (dt1 == DataType.FRAME && dt2 == DataType.FRAME) || (dt1 == DataType.SCALAR && dt2 == DataType.SCALAR && vt1 == ValueType.STRING && vt2 == ValueType.STRING))) {
        throw new HopsException("Append can only apply to two matrices, two frames, or two scalar strings!");
    }
    Lop append = null;
    if (dt1 == DataType.MATRIX || dt1 == DataType.FRAME) {
        long rlen = cbind ? getInput().get(0).getDim1() : (getInput().get(0).dimsKnown() && getInput().get(1).dimsKnown()) ? getInput().get(0).getDim1() + getInput().get(1).getDim1() : -1;
        long clen = cbind ? ((getInput().get(0).dimsKnown() && getInput().get(1).dimsKnown()) ? getInput().get(0).getDim2() + getInput().get(1).getDim2() : -1) : getInput().get(0).getDim2();
        if (et == ExecType.MR) {
            append = constructMRAppendLop(getInput().get(0), getInput().get(1), getDataType(), getValueType(), cbind, this);
        } else if (et == ExecType.SPARK) {
            append = constructSPAppendLop(getInput().get(0), getInput().get(1), getDataType(), getValueType(), cbind, this);
            append.getOutputParameters().setDimensions(rlen, clen, getRowsInBlock(), getColsInBlock(), getNnz());
        } else //CP
        {
            //offset 1st input
            Lop offset = createOffsetLop(getInput().get(0), cbind);
            append = new AppendCP(getInput().get(0).constructLops(), getInput().get(1).constructLops(), offset, getDataType(), getValueType(), cbind);
            append.getOutputParameters().setDimensions(rlen, clen, getRowsInBlock(), getColsInBlock(), getNnz());
        }
    } else //SCALAR-STRING and SCALAR-STRING (always CP)
    {
        append = new AppendCP(getInput().get(0).constructLops(), getInput().get(1).constructLops(), Data.createLiteralLop(ValueType.INT, "-1"), getDataType(), getValueType(), cbind);
        append.getOutputParameters().setDimensions(0, 0, -1, -1, -1);
    }
    setLineNumbers(append);
    setLops(append);
}

Example 10 with ValueType

use of org.apache.sysml.parser.Expression.ValueType in project incubator-systemml by apache.

the class DataPartitioner method createPartitionedMatrixObject.

public MatrixObject createPartitionedMatrixObject(MatrixObject in, String fnameNew, boolean force) throws DMLRuntimeException {
    ValueType vt = in.getValueType();
    String varname = in.getVarName();
    MatrixObject out = new MatrixObject(vt, fnameNew);
    out.setDataType(DataType.MATRIX);
    out.setVarName(varname + NAME_SUFFIX);
    return createPartitionedMatrixObject(in, out, force);
}