Examples with IndexRange org.apache.sysml.runtime.util.IndexRange used on opensource projects

Example 1 with IndexRange

use of org.apache.sysml.runtime.util.IndexRange in project incubator-systemml by apache.

the class PartitionedBlock method sliceOperations.

/**
	 * Utility for slice operations over partitioned matrices, where the index range can cover
	 * multiple blocks. The result is always a single result matrix block. All semantics are 
	 * equivalent to the core matrix block slice operations. 
	 * 
	 * @param rl row lower bound
	 * @param ru row upper bound
	 * @param cl column lower bound
	 * @param cu column upper bound
	 * @param block block object
	 * @return block object
	 * @throws DMLRuntimeException if DMLRuntimeException occurs
	 */
@SuppressWarnings("unchecked")
public T sliceOperations(long rl, long ru, long cl, long cu, T block) throws DMLRuntimeException {
    int lrl = (int) rl;
    int lru = (int) ru;
    int lcl = (int) cl;
    int lcu = (int) cu;
    ArrayList<Pair<?, ?>> allBlks = (ArrayList<Pair<?, ?>>) CacheBlockFactory.getPairList(block);
    int start_iix = (lrl - 1) / _brlen + 1;
    int end_iix = (lru - 1) / _brlen + 1;
    int start_jix = (lcl - 1) / _bclen + 1;
    int end_jix = (lcu - 1) / _bclen + 1;
    for (int iix = start_iix; iix <= end_iix; iix++) for (int jix = start_jix; jix <= end_jix; jix++) {
        IndexRange ixrange = new IndexRange(rl, ru, cl, cu);
        allBlks.addAll(OperationsOnMatrixValues.performSlice(ixrange, _brlen, _bclen, iix, jix, getBlock(iix, jix)));
    }
    if (allBlks.size() == 1) {
        return (T) allBlks.get(0).getValue();
    } else {
        //allocate output matrix
        Constructor<?> constr;
        try {
            constr = block.getClass().getConstructor(int.class, int.class, boolean.class);
            T ret = (T) constr.newInstance(lru - lrl + 1, lcu - lcl + 1, false);
            for (Pair<?, ?> kv : allBlks) {
                ret.merge((T) kv.getValue(), false);
            }
            return ret;
        } catch (Exception e) {
            throw new DMLRuntimeException(e);
        }
    }
}

Example 2 with IndexRange

use of org.apache.sysml.runtime.util.IndexRange in project incubator-systemml by apache.

the class FrameIndexingSPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    SparkExecutionContext sec = (SparkExecutionContext) ec;
    String opcode = getOpcode();
    // get indexing range
    long rl = ec.getScalarInput(rowLower.getName(), rowLower.getValueType(), rowLower.isLiteral()).getLongValue();
    long ru = ec.getScalarInput(rowUpper.getName(), rowUpper.getValueType(), rowUpper.isLiteral()).getLongValue();
    long cl = ec.getScalarInput(colLower.getName(), colLower.getValueType(), colLower.isLiteral()).getLongValue();
    long cu = ec.getScalarInput(colUpper.getName(), colUpper.getValueType(), colUpper.isLiteral()).getLongValue();
    IndexRange ixrange = new IndexRange(rl, ru, cl, cu);
    // right indexing
    if (opcode.equalsIgnoreCase(RightIndex.OPCODE)) {
        // update and check output dimensions
        MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(input1.getName());
        MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName());
        mcOut.set(ru - rl + 1, cu - cl + 1, mcIn.getRowsPerBlock(), mcIn.getColsPerBlock());
        checkValidOutputDimensions(mcOut);
        // execute right indexing operation (partitioning-preserving if possible)
        JavaPairRDD<Long, FrameBlock> in1 = sec.getFrameBinaryBlockRDDHandleForVariable(input1.getName());
        JavaPairRDD<Long, FrameBlock> out = null;
        if (isPartitioningPreservingRightIndexing(mcIn, ixrange)) {
            out = in1.mapPartitionsToPair(new SliceBlockPartitionFunction(ixrange, mcOut), true);
        } else {
            out = in1.filter(new IsFrameBlockInRange(rl, ru, mcOut)).mapToPair(new SliceBlock(ixrange, mcOut));
        }
        // put output RDD handle into symbol table
        sec.setRDDHandleForVariable(output.getName(), out);
        sec.addLineageRDD(output.getName(), input1.getName());
        // update schema of output with subset of input schema
        sec.getFrameObject(output.getName()).setSchema(sec.getFrameObject(input1.getName()).getSchema((int) cl, (int) cu));
    } else // left indexing
    if (opcode.equalsIgnoreCase(LeftIndex.OPCODE) || opcode.equalsIgnoreCase("mapLeftIndex")) {
        JavaPairRDD<Long, FrameBlock> in1 = sec.getFrameBinaryBlockRDDHandleForVariable(input1.getName());
        PartitionedBroadcast<FrameBlock> broadcastIn2 = null;
        JavaPairRDD<Long, FrameBlock> in2 = null;
        JavaPairRDD<Long, FrameBlock> out = null;
        // update and check output dimensions
        MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName());
        MatrixCharacteristics mcLeft = ec.getMatrixCharacteristics(input1.getName());
        mcOut.set(mcLeft.getRows(), mcLeft.getCols(), mcLeft.getRowsPerBlock(), mcLeft.getColsPerBlock());
        checkValidOutputDimensions(mcOut);
        // note: always frame rhs, scalars are preprocessed via cast to 1x1 frame
        MatrixCharacteristics mcRight = ec.getMatrixCharacteristics(input2.getName());
        // sanity check matching index range and rhs dimensions
        if (!mcRight.dimsKnown()) {
            throw new DMLRuntimeException("The right input frame dimensions are not specified for FrameIndexingSPInstruction");
        }
        if (!(ru - rl + 1 == mcRight.getRows() && cu - cl + 1 == mcRight.getCols())) {
            throw new DMLRuntimeException("Invalid index range of leftindexing: [" + rl + ":" + ru + "," + cl + ":" + cu + "] vs [" + mcRight.getRows() + "x" + mcRight.getCols() + "].");
        }
        if (opcode.equalsIgnoreCase("mapLeftIndex")) {
            broadcastIn2 = sec.getBroadcastForFrameVariable(input2.getName());
            // partitioning-preserving mappartitions (key access required for broadcast loopkup)
            out = in1.mapPartitionsToPair(new LeftIndexPartitionFunction(broadcastIn2, ixrange, mcOut), true);
        } else {
            // general case
            // zero-out lhs
            in1 = in1.flatMapToPair(new ZeroOutLHS(false, ixrange, mcLeft));
            // slice rhs, shift and merge with lhs
            in2 = sec.getFrameBinaryBlockRDDHandleForVariable(input2.getName()).flatMapToPair(new SliceRHSForLeftIndexing(ixrange, mcLeft));
            out = FrameRDDAggregateUtils.mergeByKey(in1.union(in2));
        }
        sec.setRDDHandleForVariable(output.getName(), out);
        sec.addLineageRDD(output.getName(), input1.getName());
        if (broadcastIn2 != null)
            sec.addLineageBroadcast(output.getName(), input2.getName());
        if (in2 != null)
            sec.addLineageRDD(output.getName(), input2.getName());
    } else
        throw new DMLRuntimeException("Invalid opcode (" + opcode + ") encountered in FrameIndexingSPInstruction.");
}

Example 3 with IndexRange

use of org.apache.sysml.runtime.util.IndexRange in project incubator-systemml by apache.

the class MatrixIndexingSPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    SparkExecutionContext sec = (SparkExecutionContext) ec;
    String opcode = getOpcode();
    // get indexing range
    long rl = ec.getScalarInput(rowLower.getName(), rowLower.getValueType(), rowLower.isLiteral()).getLongValue();
    long ru = ec.getScalarInput(rowUpper.getName(), rowUpper.getValueType(), rowUpper.isLiteral()).getLongValue();
    long cl = ec.getScalarInput(colLower.getName(), colLower.getValueType(), colLower.isLiteral()).getLongValue();
    long cu = ec.getScalarInput(colUpper.getName(), colUpper.getValueType(), colUpper.isLiteral()).getLongValue();
    IndexRange ixrange = new IndexRange(rl, ru, cl, cu);
    // right indexing
    if (opcode.equalsIgnoreCase(RightIndex.OPCODE)) {
        // update and check output dimensions
        MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(input1.getName());
        MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName());
        mcOut.set(ru - rl + 1, cu - cl + 1, mcIn.getRowsPerBlock(), mcIn.getColsPerBlock());
        mcOut.setNonZerosBound(Math.min(mcOut.getLength(), mcIn.getNonZerosBound()));
        checkValidOutputDimensions(mcOut);
        // execute right indexing operation (partitioning-preserving if possible)
        JavaPairRDD<MatrixIndexes, MatrixBlock> in1 = sec.getBinaryBlockRDDHandleForVariable(input1.getName());
        if (isSingleBlockLookup(mcIn, ixrange)) {
            sec.setMatrixOutput(output.getName(), singleBlockIndexing(in1, mcIn, mcOut, ixrange), getExtendedOpcode());
        } else if (isMultiBlockLookup(in1, mcIn, mcOut, ixrange)) {
            sec.setMatrixOutput(output.getName(), multiBlockIndexing(in1, mcIn, mcOut, ixrange), getExtendedOpcode());
        } else {
            // rdd output for general case
            JavaPairRDD<MatrixIndexes, MatrixBlock> out = generalCaseRightIndexing(in1, mcIn, mcOut, ixrange, _aggType);
            // put output RDD handle into symbol table
            sec.setRDDHandleForVariable(output.getName(), out);
            sec.addLineageRDD(output.getName(), input1.getName());
        }
    } else // left indexing
    if (opcode.equalsIgnoreCase(LeftIndex.OPCODE) || opcode.equalsIgnoreCase("mapLeftIndex")) {
        String rddVar = (_type == LixCacheType.LEFT) ? input2.getName() : input1.getName();
        String bcVar = (_type == LixCacheType.LEFT) ? input1.getName() : input2.getName();
        JavaPairRDD<MatrixIndexes, MatrixBlock> in1 = sec.getBinaryBlockRDDHandleForVariable(rddVar);
        PartitionedBroadcast<MatrixBlock> broadcastIn2 = null;
        JavaPairRDD<MatrixIndexes, MatrixBlock> in2 = null;
        JavaPairRDD<MatrixIndexes, MatrixBlock> out = null;
        // update and check output dimensions
        MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName());
        MatrixCharacteristics mcLeft = ec.getMatrixCharacteristics(input1.getName());
        mcOut.set(mcLeft.getRows(), mcLeft.getCols(), mcLeft.getRowsPerBlock(), mcLeft.getColsPerBlock());
        checkValidOutputDimensions(mcOut);
        // note: always matrix rhs, scalars are preprocessed via cast to 1x1 matrix
        MatrixCharacteristics mcRight = ec.getMatrixCharacteristics(input2.getName());
        // sanity check matching index range and rhs dimensions
        if (!mcRight.dimsKnown()) {
            throw new DMLRuntimeException("The right input matrix dimensions are not specified for MatrixIndexingSPInstruction");
        }
        if (!(ru - rl + 1 == mcRight.getRows() && cu - cl + 1 == mcRight.getCols())) {
            throw new DMLRuntimeException("Invalid index range of leftindexing: [" + rl + ":" + ru + "," + cl + ":" + cu + "] vs [" + mcRight.getRows() + "x" + mcRight.getCols() + "].");
        }
        if (opcode.equalsIgnoreCase("mapLeftIndex")) {
            broadcastIn2 = sec.getBroadcastForVariable(bcVar);
            // partitioning-preserving mappartitions (key access required for broadcast loopkup)
            out = in1.mapPartitionsToPair(new LeftIndexPartitionFunction(broadcastIn2, ixrange, _type, mcOut), true);
        } else {
            // general case
            // zero-out lhs
            in1 = in1.mapToPair(new ZeroOutLHS(false, ixrange, mcLeft));
            // slice rhs, shift and merge with lhs
            in2 = sec.getBinaryBlockRDDHandleForVariable(input2.getName()).flatMapToPair(new SliceRHSForLeftIndexing(ixrange, mcLeft));
            out = RDDAggregateUtils.mergeByKey(in1.union(in2));
        }
        sec.setRDDHandleForVariable(output.getName(), out);
        sec.addLineageRDD(output.getName(), rddVar);
        if (broadcastIn2 != null)
            sec.addLineageBroadcast(output.getName(), bcVar);
        if (in2 != null)
            sec.addLineageRDD(output.getName(), input2.getName());
    } else
        throw new DMLRuntimeException("Invalid opcode (" + opcode + ") encountered in MatrixIndexingSPInstruction.");
}

Example 4 with IndexRange

use of org.apache.sysml.runtime.util.IndexRange in project incubator-systemml by apache.

the class MatrixIndexingCPFileInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    String opcode = getOpcode();
    IndexRange ixrange = getIndexRange(ec).add(1);
    MatrixObject mo = ec.getMatrixObject(input1.getName());
    if (mo.isPartitioned() && opcode.equalsIgnoreCase(RightIndex.OPCODE)) {
        MetaDataFormat meta = (MetaDataFormat) mo.getMetaData();
        MatrixCharacteristics mc = meta.getMatrixCharacteristics();
        String pfname = mo.getPartitionFileName(ixrange, mc.getRowsPerBlock(), mc.getColsPerBlock());
        if (MapReduceTool.existsFileOnHDFS(pfname)) {
            // create output matrix object
            MatrixObject mobj = new MatrixObject(mo.getValueType(), pfname);
            MatrixCharacteristics mcNew = null;
            switch(mo.getPartitionFormat()) {
                case ROW_WISE:
                    mcNew = new MatrixCharacteristics(1, mc.getCols(), mc.getRowsPerBlock(), mc.getColsPerBlock());
                    break;
                case ROW_BLOCK_WISE_N:
                    mcNew = new MatrixCharacteristics(mo.getPartitionSize(), mc.getCols(), mc.getRowsPerBlock(), mc.getColsPerBlock());
                    break;
                case COLUMN_WISE:
                    mcNew = new MatrixCharacteristics(mc.getRows(), 1, mc.getRowsPerBlock(), mc.getColsPerBlock());
                    break;
                case COLUMN_BLOCK_WISE_N:
                    mcNew = new MatrixCharacteristics(mc.getRows(), mo.getPartitionSize(), mc.getRowsPerBlock(), mc.getColsPerBlock());
                    break;
                default:
                    throw new DMLRuntimeException("Unsupported partition format for CP_FILE " + RightIndex.OPCODE + ": " + mo.getPartitionFormat());
            }
            MetaDataFormat metaNew = new MetaDataFormat(mcNew, meta.getOutputInfo(), meta.getInputInfo());
            mobj.setMetaData(metaNew);
            // put output object into symbol table
            ec.setVariable(output.getName(), mobj);
        } else {
            // will return an empty matrix partition
            MatrixBlock resultBlock = mo.readMatrixPartition(ixrange);
            ec.setMatrixOutput(output.getName(), resultBlock, getExtendedOpcode());
        }
    } else {
        throw new DMLRuntimeException("Invalid opcode or index predicate for MatrixIndexingCPFileInstruction: " + instString);
    }
}

Example 5 with IndexRange

use of org.apache.sysml.runtime.util.IndexRange in project incubator-systemml by apache.

the class RangeBasedReIndexInstruction method parseInstruction.

public static RangeBasedReIndexInstruction parseInstruction(String str) {
    InstructionUtils.checkNumFields(str, 8);
    String[] parts = InstructionUtils.getInstructionParts(str);
    String opcode = parts[0];
    boolean forLeft = false;
    if (opcode.equalsIgnoreCase(RightIndex.OPCODE + "ForLeft"))
        forLeft = true;
    else if (!opcode.equalsIgnoreCase(RightIndex.OPCODE))
        throw new DMLRuntimeException("Unknown opcode while parsing a Select: " + str);
    byte in = Byte.parseByte(parts[1]);
    IndexRange rng = new IndexRange(UtilFunctions.parseToLong(parts[2]), UtilFunctions.parseToLong(parts[3]), UtilFunctions.parseToLong(parts[4]), UtilFunctions.parseToLong(parts[5]));
    byte out = Byte.parseByte(parts[6]);
    long rlen = Long.parseLong(parts[7]);
    long clen = Long.parseLong(parts[8]);
    return new RangeBasedReIndexInstruction(new ReIndexOperator(), in, out, rng, forLeft, rlen, clen, str);
}