Examples with ReorgOperator org.apache.sysml.runtime.matrix.operators.ReorgOperator used on opensource projects

Example 6 with ReorgOperator

use of org.apache.sysml.runtime.matrix.operators.ReorgOperator in project incubator-systemml by apache.

the class ReorgCPInstruction method parseInstruction.

public static ReorgCPInstruction parseInstruction(String str) throws DMLRuntimeException {
    CPOperand in = new CPOperand("", ValueType.UNKNOWN, DataType.UNKNOWN);
    CPOperand out = new CPOperand("", ValueType.UNKNOWN, DataType.UNKNOWN);
    String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
    String opcode = parts[0];
    if (opcode.equalsIgnoreCase("r'")) {
        InstructionUtils.checkNumFields(str, 2, 3);
        in.split(parts[1]);
        out.split(parts[2]);
        int k = Integer.parseInt(parts[3]);
        return new ReorgCPInstruction(new ReorgOperator(SwapIndex.getSwapIndexFnObject(), k), in, out, opcode, str);
    } else if (opcode.equalsIgnoreCase("rev")) {
        //max 2 operands
        parseUnaryInstruction(str, in, out);
        return new ReorgCPInstruction(new ReorgOperator(RevIndex.getRevIndexFnObject()), in, out, opcode, str);
    } else if (opcode.equalsIgnoreCase("rdiag")) {
        //max 2 operands
        parseUnaryInstruction(str, in, out);
        return new ReorgCPInstruction(new ReorgOperator(DiagIndex.getDiagIndexFnObject()), in, out, opcode, str);
    } else if (opcode.equalsIgnoreCase("rsort")) {
        InstructionUtils.checkNumFields(parts, 5);
        in.split(parts[1]);
        out.split(parts[5]);
        CPOperand col = new CPOperand(parts[2]);
        CPOperand desc = new CPOperand(parts[3]);
        CPOperand ixret = new CPOperand(parts[4]);
        return new ReorgCPInstruction(new ReorgOperator(SortIndex.getSortIndexFnObject(1, false, false)), in, col, desc, ixret, out, opcode, str);
    } else {
        throw new DMLRuntimeException("Unknown opcode while parsing a ReorgInstruction: " + str);
    }
}

Example 7 with ReorgOperator

use of org.apache.sysml.runtime.matrix.operators.ReorgOperator in project incubator-systemml by apache.

the class AppendCPInstruction method parseInstruction.

public static AppendCPInstruction parseInstruction(String str) throws DMLRuntimeException {
    String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
    InstructionUtils.checkNumFields(parts, 5);
    String opcode = parts[0];
    CPOperand in1 = new CPOperand(parts[1]);
    CPOperand in2 = new CPOperand(parts[2]);
    CPOperand in3 = new CPOperand(parts[3]);
    CPOperand out = new CPOperand(parts[4]);
    boolean cbind = Boolean.parseBoolean(parts[5]);
    AppendType type = (in1.getDataType() != DataType.MATRIX && in1.getDataType() != DataType.FRAME) ? AppendType.STRING : cbind ? AppendType.CBIND : AppendType.RBIND;
    if (!opcode.equalsIgnoreCase("append"))
        throw new DMLRuntimeException("Unknown opcode while parsing a AppendCPInstruction: " + str);
    Operator op = new ReorgOperator(OffsetColumnIndex.getOffsetColumnIndexFnObject(-1));
    if (type == AppendType.STRING)
        return new ScalarAppendCPInstruction(op, in1, in2, in3, out, type, opcode, str);
    else if (in1.getDataType() == DataType.MATRIX)
        return new MatrixAppendCPInstruction(op, in1, in2, in3, out, type, opcode, str);
    else
        //DataType.FRAME
        return new FrameAppendCPInstruction(op, in1, in2, in3, out, type, opcode, str);
}

Example 8 with ReorgOperator

use of org.apache.sysml.runtime.matrix.operators.ReorgOperator in project incubator-systemml by apache.

the class RDDSortUtils method sortDataByValMemSort.

/**
	 * This function collects and sorts value column in memory and then broadcasts it. 
	 * 
	 * @param val value as {@code JavaPairRDD<MatrixIndexes, MatrixBlock>}
	 * @param data data as {@code JavaPairRDD<MatrixIndexes, MatrixBlock>}
	 * @param asc if true, sort ascending
	 * @param rlen number of rows
	 * @param clen number of columns
	 * @param brlen number of rows in a block
	 * @param bclen number of columns in a block
	 * @param sec spark execution context
	 * @param r_op reorg operator
	 * @return data as {@code JavaPairRDD<MatrixIndexes, MatrixBlock>}
	 * @throws DMLRuntimeException if DMLRuntimeException occurs
	 */
public static JavaPairRDD<MatrixIndexes, MatrixBlock> sortDataByValMemSort(JavaPairRDD<MatrixIndexes, MatrixBlock> val, JavaPairRDD<MatrixIndexes, MatrixBlock> data, boolean asc, long rlen, long clen, int brlen, int bclen, SparkExecutionContext sec, ReorgOperator r_op) throws DMLRuntimeException {
    //collect orderby column for in-memory sorting
    MatrixBlock inMatBlock = SparkExecutionContext.toMatrixBlock(val, (int) rlen, 1, brlen, bclen, -1);
    //in-memory sort operation (w/ index return: source index in target position)
    ReorgOperator lrop = new ReorgOperator(SortIndex.getSortIndexFnObject(1, !asc, true));
    MatrixBlock sortedIx = (MatrixBlock) inMatBlock.reorgOperations(lrop, new MatrixBlock(), -1, -1, -1);
    //flip sort indices from <source ix in target pos> to <target ix in source pos>
    MatrixBlock sortedIxSrc = new MatrixBlock(sortedIx.getNumRows(), 1, false);
    for (int i = 0; i < sortedIx.getNumRows(); i++) sortedIxSrc.quickSetValue((int) sortedIx.quickGetValue(i, 0) - 1, 0, i + 1);
    //broadcast index vector
    PartitionedBlock<MatrixBlock> pmb = new PartitionedBlock<MatrixBlock>(sortedIxSrc, brlen, bclen);
    Broadcast<PartitionedBlock<MatrixBlock>> _pmb = sec.getSparkContext().broadcast(pmb);
    //sort data with broadcast index vector
    JavaPairRDD<MatrixIndexes, RowMatrixBlock> ret = data.mapPartitionsToPair(new ShuffleMatrixBlockRowsInMemFunction(rlen, brlen, _pmb));
    return RDDAggregateUtils.mergeRowsByKey(ret);
}

Example 9 with ReorgOperator

use of org.apache.sysml.runtime.matrix.operators.ReorgOperator in project incubator-systemml by apache.

the class LibMatrixMult method prepMatrixMultTransposeSelfInput.

private static MatrixBlock prepMatrixMultTransposeSelfInput(MatrixBlock m1, boolean leftTranspose) throws DMLRuntimeException {
    MatrixBlock ret = m1;
    if (//X%*%t(X) SPARSE MATRIX
    !leftTranspose && m1.sparse && m1.rlen > 1) {
        //directly via LibMatrixReorg in order to prevent sparsity change
        MatrixBlock tmpBlock = new MatrixBlock(m1.clen, m1.rlen, m1.sparse);
        LibMatrixReorg.reorg(m1, tmpBlock, new ReorgOperator(SwapIndex.getSwapIndexFnObject()));
        ret = tmpBlock;
    }
    return ret;
}

Example 10 with ReorgOperator

use of org.apache.sysml.runtime.matrix.operators.ReorgOperator in project incubator-systemml by apache.

the class ReorgCPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) throws DMLRuntimeException {
    //acquire inputs
    MatrixBlock matBlock = ec.getMatrixInput(input1.getName());
    ReorgOperator r_op = (ReorgOperator) _optr;
    if (r_op.fn instanceof SortIndex) {
        //additional attributes for sort
        int col = (int) ec.getScalarInput(_col.getName(), _col.getValueType(), _col.isLiteral()).getLongValue();
        boolean desc = ec.getScalarInput(_desc.getName(), _desc.getValueType(), _desc.isLiteral()).getBooleanValue();
        boolean ixret = ec.getScalarInput(_ixret.getName(), _ixret.getValueType(), _ixret.isLiteral()).getBooleanValue();
        r_op.fn = SortIndex.getSortIndexFnObject(col, desc, ixret);
    }
    //execute operation
    MatrixBlock soresBlock = (MatrixBlock) (matBlock.reorgOperations(r_op, new MatrixBlock(), 0, 0, 0));
    //release inputs/outputs
    ec.releaseMatrixInput(input1.getName());
    ec.setMatrixOutput(output.getName(), soresBlock);
}