Examples with DataType org.apache.sysml.parser.Expression.DataType used on opensource projects

Example 56 with DataType

use of org.apache.sysml.parser.Expression.DataType in project systemml by apache.

the class ProgramConverter method serializeDataObject.

public static String serializeDataObject(String key, Data dat) {
    // SCHEMA: <name>|<datatype>|<valuetype>|value
    // (scalars are serialize by value, matrices by filename)
    StringBuilder sb = new StringBuilder();
    // prepare data for serialization
    String name = key;
    DataType datatype = dat.getDataType();
    ValueType valuetype = dat.getValueType();
    String value = null;
    String[] matrixMetaData = null;
    switch(datatype) {
        case SCALAR:
            ScalarObject so = (ScalarObject) dat;
            // name = so.getName();
            value = so.getStringValue();
            break;
        case MATRIX:
            MatrixObject mo = (MatrixObject) dat;
            MetaDataFormat md = (MetaDataFormat) dat.getMetaData();
            MatrixCharacteristics mc = md.getMatrixCharacteristics();
            value = mo.getFileName();
            PartitionFormat partFormat = (mo.getPartitionFormat() != null) ? new PartitionFormat(mo.getPartitionFormat(), mo.getPartitionSize()) : PartitionFormat.NONE;
            matrixMetaData = new String[9];
            matrixMetaData[0] = String.valueOf(mc.getRows());
            matrixMetaData[1] = String.valueOf(mc.getCols());
            matrixMetaData[2] = String.valueOf(mc.getRowsPerBlock());
            matrixMetaData[3] = String.valueOf(mc.getColsPerBlock());
            matrixMetaData[4] = String.valueOf(mc.getNonZeros());
            matrixMetaData[5] = InputInfo.inputInfoToString(md.getInputInfo());
            matrixMetaData[6] = OutputInfo.outputInfoToString(md.getOutputInfo());
            matrixMetaData[7] = String.valueOf(partFormat);
            matrixMetaData[8] = String.valueOf(mo.getUpdateType());
            break;
        default:
            throw new DMLRuntimeException("Unable to serialize datatype " + datatype);
    }
    // serialize data
    sb.append(name);
    sb.append(DATA_FIELD_DELIM);
    sb.append(datatype);
    sb.append(DATA_FIELD_DELIM);
    sb.append(valuetype);
    sb.append(DATA_FIELD_DELIM);
    sb.append(value);
    if (matrixMetaData != null)
        for (int i = 0; i < matrixMetaData.length; i++) {
            sb.append(DATA_FIELD_DELIM);
            sb.append(matrixMetaData[i]);
        }
    return sb.toString();
}

Example 57 with DataType

use of org.apache.sysml.parser.Expression.DataType in project systemml by apache.

the class FrameMatrixCastingTest method runFrameCastingTest.

/**
 * @param testname
 * @param schema
 * @param wildcard
 */
private void runFrameCastingTest(String testname, boolean multColBlks, ValueType vt, ExecType et) {
    // rtplatform for MR
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(et) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        int cols = multColBlks ? cols2 : cols1;
        TestConfiguration config = getTestConfiguration(testname);
        loadTestConfiguration(config);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + testname + ".dml";
        programArgs = new String[] { "-explain", "-args", input("A"), output("B") };
        // data generation
        double[][] A = getRandomMatrix(rows, cols, -1, 1, 0.9, 7);
        DataType dtin = testname.equals(TEST_NAME1) ? DataType.FRAME : DataType.MATRIX;
        ValueType vtin = testname.equals(TEST_NAME1) ? vt : ValueType.DOUBLE;
        writeMatrixOrFrameInput(input("A"), A, rows, cols, dtin, vtin);
        // run testcase
        runTest(true, false, null, -1);
        // compare matrices
        DataType dtout = testname.equals(TEST_NAME1) ? DataType.MATRIX : DataType.FRAME;
        double[][] B = readMatrixOrFrameInput(output("B"), rows, cols, dtout);
        TestUtils.compareMatrices(A, B, rows, cols, 0);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 58 with DataType

use of org.apache.sysml.parser.Expression.DataType in project systemml by apache.

the class TemplateCell method isValidOperation.

protected static boolean isValidOperation(Hop hop) {
    // prepare indicators for binary operations
    boolean isBinaryMatrixScalar = false;
    boolean isBinaryMatrixVector = false;
    boolean isBinaryMatrixMatrix = false;
    if (hop instanceof BinaryOp && hop.getDataType().isMatrix()) {
        Hop left = hop.getInput().get(0);
        Hop right = hop.getInput().get(1);
        DataType ldt = left.getDataType();
        DataType rdt = right.getDataType();
        isBinaryMatrixScalar = (ldt.isScalar() || rdt.isScalar());
        isBinaryMatrixVector = hop.dimsKnown() && ((ldt.isMatrix() && TemplateUtils.isVectorOrScalar(right)) || (rdt.isMatrix() && TemplateUtils.isVectorOrScalar(left)));
        isBinaryMatrixMatrix = hop.dimsKnown() && HopRewriteUtils.isEqualSize(left, right) && ldt.isMatrix() && rdt.isMatrix();
    }
    // prepare indicators for ternary operations
    boolean isTernaryVectorScalarVector = false;
    boolean isTernaryMatrixScalarMatrixDense = false;
    boolean isTernaryIfElse = (HopRewriteUtils.isTernary(hop, OpOp3.IFELSE) && hop.getDataType().isMatrix());
    if (hop instanceof TernaryOp && hop.getInput().size() == 3 && hop.dimsKnown() && HopRewriteUtils.checkInputDataTypes(hop, DataType.MATRIX, DataType.SCALAR, DataType.MATRIX)) {
        Hop left = hop.getInput().get(0);
        Hop right = hop.getInput().get(2);
        isTernaryVectorScalarVector = TemplateUtils.isVector(left) && TemplateUtils.isVector(right);
        isTernaryMatrixScalarMatrixDense = HopRewriteUtils.isEqualSize(left, right) && !HopRewriteUtils.isSparse(left) && !HopRewriteUtils.isSparse(right);
    }
    // check supported unary, binary, ternary operations
    return hop.getDataType() == DataType.MATRIX && TemplateUtils.isOperationSupported(hop) && (hop instanceof UnaryOp || isBinaryMatrixScalar || isBinaryMatrixVector || isBinaryMatrixMatrix || isTernaryVectorScalarVector || isTernaryMatrixScalarMatrixDense || isTernaryIfElse || (hop instanceof ParameterizedBuiltinOp && ((ParameterizedBuiltinOp) hop).getOp() == ParamBuiltinOp.REPLACE));
}

Example 59 with DataType

use of org.apache.sysml.parser.Expression.DataType in project systemml by apache.

the class HopRewriteUtils method createAggUnaryOp.

public static AggUnaryOp createAggUnaryOp(Hop input, AggOp op, Direction dir) {
    DataType dt = (dir == Direction.RowCol) ? DataType.SCALAR : input.getDataType();
    AggUnaryOp auop = new AggUnaryOp(input.getName(), dt, input.getValueType(), op, dir, input);
    auop.setOutputBlocksizes(input.getRowsInBlock(), input.getColsInBlock());
    copyLineNumbers(input, auop);
    auop.refreshSizeInformation();
    return auop;
}