Examples with AppendG org.apache.sysml.lops.AppendG used on opensource projects

Search in sources :

Example 1 with AppendG

use of org.apache.sysml.lops.AppendG in project incubator-systemml by apache.

the class BinaryOp method constructSPAppendLop.

public static Lop constructSPAppendLop(Hop left, Hop right, DataType dt, ValueType vt, boolean cbind, Hop current) {
    Lop ret = null;
    // offset 1st input
    Lop offset = createOffsetLop(left, cbind);
    AppendMethod am = optFindAppendSPMethod(left.getDim1(), left.getDim2(), right.getDim1(), right.getDim2(), right.getRowsInBlock(), right.getColsInBlock(), right.getNnz(), cbind, dt);
    switch(am) {
        case // special case map-only append
        MR_MAPPEND:
            {
                ret = new AppendM(left.constructLops(), right.constructLops(), offset, current.getDataType(), current.getValueType(), cbind, false, ExecType.SPARK);
                break;
            }
        case // special case reduce append w/ one column block
        MR_RAPPEND:
            {
                ret = new AppendR(left.constructLops(), right.constructLops(), current.getDataType(), current.getValueType(), cbind, ExecType.SPARK);
                break;
            }
        case MR_GAPPEND:
            {
                // offset second input
                Lop offset2 = createOffsetLop(right, cbind);
                ret = new AppendG(left.constructLops(), right.constructLops(), offset, offset2, current.getDataType(), current.getValueType(), cbind, ExecType.SPARK);
                break;
            }
        case SP_GAlignedAppend:
            {
                ret = new AppendGAlignedSP(left.constructLops(), right.constructLops(), offset, current.getDataType(), current.getValueType(), cbind);
                break;
            }
        default:
            throw new HopsException("Invalid SP append method: " + am);
    }
    ret.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
    return ret;
}
Also used : AppendG(org.apache.sysml.lops.AppendG) AppendR(org.apache.sysml.lops.AppendR) Lop(org.apache.sysml.lops.Lop) AppendGAlignedSP(org.apache.sysml.lops.AppendGAlignedSP) AppendM(org.apache.sysml.lops.AppendM)

Example 2 with AppendG

use of org.apache.sysml.lops.AppendG in project systemml by apache.

the class BinaryOp method constructMRAppendLop.

/**
 * General case binary append.
 *
 * @param left high-level operator left
 * @param right high-level operator right
 * @param dt data type
 * @param vt value type
 * @param cbind true if cbind
 * @param current current high-level operator
 * @return low-level operator
 */
public static Lop constructMRAppendLop(Hop left, Hop right, DataType dt, ValueType vt, boolean cbind, Hop current) {
    Lop ret = null;
    long m1_dim1 = left.getDim1();
    long m1_dim2 = left.getDim2();
    long m2_dim1 = right.getDim1();
    long m2_dim2 = right.getDim2();
    // output rows
    long m3_dim1 = cbind ? m1_dim1 : ((m1_dim1 >= 0 && m2_dim1 >= 0) ? (m1_dim1 + m2_dim1) : -1);
    // output cols
    long m3_dim2 = cbind ? ((m1_dim2 >= 0 && m2_dim2 >= 0) ? (m1_dim2 + m2_dim2) : -1) : m1_dim2;
    // output nnz
    long m3_nnz = (left.getNnz() > 0 && right.getNnz() > 0) ? (left.getNnz() + right.getNnz()) : -1;
    long brlen = left.getRowsInBlock();
    long bclen = left.getColsInBlock();
    // offset 1st input
    Lop offset = createOffsetLop(left, cbind);
    AppendMethod am = optFindAppendMethod(m1_dim1, m1_dim2, m2_dim1, m2_dim2, brlen, bclen, cbind);
    switch(am) {
        case // special case map-only append
        MR_MAPPEND:
            {
                boolean needPart = requiresPartitioning(right);
                // pre partitioning
                Lop dcInput = right.constructLops();
                if (needPart) {
                    // right side in distributed cache
                    ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(right.getDim1(), right.getDim2(), OptimizerUtils.getSparsity(right.getDim1(), right.getDim2(), right.getNnz())) < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : // operator selection
                    ExecType.MR;
                    dcInput = new DataPartition(dcInput, DataType.MATRIX, ValueType.DOUBLE, etPart, PDataPartitionFormat.ROW_BLOCK_WISE_N);
                    dcInput.getOutputParameters().setDimensions(right.getDim1(), right.getDim2(), right.getRowsInBlock(), right.getColsInBlock(), right.getNnz());
                    dcInput.setAllPositions(right.getFilename(), right.getBeginLine(), right.getBeginColumn(), right.getEndLine(), right.getEndColumn());
                }
                AppendM appM = new AppendM(left.constructLops(), dcInput, offset, dt, vt, cbind, needPart, ExecType.MR);
                appM.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
                appM.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
                ret = appM;
                break;
            }
        case // special case reduce append w/ one column block
        MR_RAPPEND:
            {
                // group
                Group group1 = new Group(left.constructLops(), Group.OperationTypes.Sort, DataType.MATRIX, vt);
                group1.getOutputParameters().setDimensions(m1_dim1, m1_dim2, brlen, bclen, left.getNnz());
                group1.setAllPositions(left.getFilename(), left.getBeginLine(), left.getBeginColumn(), left.getEndLine(), left.getEndColumn());
                Group group2 = new Group(right.constructLops(), Group.OperationTypes.Sort, DataType.MATRIX, vt);
                group1.getOutputParameters().setDimensions(m2_dim1, m2_dim2, brlen, bclen, right.getNnz());
                group1.setAllPositions(right.getFilename(), right.getBeginLine(), right.getBeginColumn(), right.getEndLine(), right.getEndColumn());
                AppendR appR = new AppendR(group1, group2, dt, vt, cbind, ExecType.MR);
                appR.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
                appR.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
                ret = appR;
                break;
            }
        case MR_GAPPEND:
            {
                // general case: map expand append, reduce aggregate
                // offset second input
                Lop offset2 = createOffsetLop(right, cbind);
                AppendG appG = new AppendG(left.constructLops(), right.constructLops(), offset, offset2, dt, vt, cbind, ExecType.MR);
                appG.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
                appG.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
                // group
                Group group1 = new Group(appG, Group.OperationTypes.Sort, DataType.MATRIX, vt);
                group1.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
                group1.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
                // aggregate
                Aggregate agg1 = new Aggregate(group1, Aggregate.OperationTypes.Sum, DataType.MATRIX, vt, ExecType.MR);
                agg1.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
                agg1.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
                ret = agg1;
                break;
            }
        default:
            throw new HopsException("Invalid MR append method: " + am);
    }
    return ret;
}
Also used : AppendG(org.apache.sysml.lops.AppendG) Group(org.apache.sysml.lops.Group) AppendR(org.apache.sysml.lops.AppendR) ExecType(org.apache.sysml.lops.LopProperties.ExecType) Lop(org.apache.sysml.lops.Lop) PartialAggregate(org.apache.sysml.lops.PartialAggregate) Aggregate(org.apache.sysml.lops.Aggregate) DataPartition(org.apache.sysml.lops.DataPartition) AppendM(org.apache.sysml.lops.AppendM)

Example 3 with AppendG

use of org.apache.sysml.lops.AppendG in project incubator-systemml by apache.

the class BinaryOp method constructMRAppendLop.

/**
 * General case binary append.
 *
 * @param left high-level operator left
 * @param right high-level operator right
 * @param dt data type
 * @param vt value type
 * @param cbind true if cbind
 * @param current current high-level operator
 * @return low-level operator
 */
public static Lop constructMRAppendLop(Hop left, Hop right, DataType dt, ValueType vt, boolean cbind, Hop current) {
    Lop ret = null;
    long m1_dim1 = left.getDim1();
    long m1_dim2 = left.getDim2();
    long m2_dim1 = right.getDim1();
    long m2_dim2 = right.getDim2();
    // output rows
    long m3_dim1 = cbind ? m1_dim1 : ((m1_dim1 >= 0 && m2_dim1 >= 0) ? (m1_dim1 + m2_dim1) : -1);
    // output cols
    long m3_dim2 = cbind ? ((m1_dim2 >= 0 && m2_dim2 >= 0) ? (m1_dim2 + m2_dim2) : -1) : m1_dim2;
    // output nnz
    long m3_nnz = (left.getNnz() > 0 && right.getNnz() > 0) ? (left.getNnz() + right.getNnz()) : -1;
    long brlen = left.getRowsInBlock();
    long bclen = left.getColsInBlock();
    // offset 1st input
    Lop offset = createOffsetLop(left, cbind);
    AppendMethod am = optFindAppendMethod(m1_dim1, m1_dim2, m2_dim1, m2_dim2, brlen, bclen, cbind);
    switch(am) {
        case // special case map-only append
        MR_MAPPEND:
            {
                boolean needPart = requiresPartitioning(right);
                // pre partitioning
                Lop dcInput = right.constructLops();
                if (needPart) {
                    // right side in distributed cache
                    ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(right.getDim1(), right.getDim2(), OptimizerUtils.getSparsity(right.getDim1(), right.getDim2(), right.getNnz())) < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : // operator selection
                    ExecType.MR;
                    dcInput = new DataPartition(dcInput, DataType.MATRIX, ValueType.DOUBLE, etPart, PDataPartitionFormat.ROW_BLOCK_WISE_N);
                    dcInput.getOutputParameters().setDimensions(right.getDim1(), right.getDim2(), right.getRowsInBlock(), right.getColsInBlock(), right.getNnz());
                    dcInput.setAllPositions(right.getFilename(), right.getBeginLine(), right.getBeginColumn(), right.getEndLine(), right.getEndColumn());
                }
                AppendM appM = new AppendM(left.constructLops(), dcInput, offset, dt, vt, cbind, needPart, ExecType.MR);
                appM.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
                appM.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
                ret = appM;
                break;
            }
        case // special case reduce append w/ one column block
        MR_RAPPEND:
            {
                // group
                Group group1 = new Group(left.constructLops(), Group.OperationTypes.Sort, DataType.MATRIX, vt);
                group1.getOutputParameters().setDimensions(m1_dim1, m1_dim2, brlen, bclen, left.getNnz());
                group1.setAllPositions(left.getFilename(), left.getBeginLine(), left.getBeginColumn(), left.getEndLine(), left.getEndColumn());
                Group group2 = new Group(right.constructLops(), Group.OperationTypes.Sort, DataType.MATRIX, vt);
                group1.getOutputParameters().setDimensions(m2_dim1, m2_dim2, brlen, bclen, right.getNnz());
                group1.setAllPositions(right.getFilename(), right.getBeginLine(), right.getBeginColumn(), right.getEndLine(), right.getEndColumn());
                AppendR appR = new AppendR(group1, group2, dt, vt, cbind, ExecType.MR);
                appR.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
                appR.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
                ret = appR;
                break;
            }
        case MR_GAPPEND:
            {
                // general case: map expand append, reduce aggregate
                // offset second input
                Lop offset2 = createOffsetLop(right, cbind);
                AppendG appG = new AppendG(left.constructLops(), right.constructLops(), offset, offset2, dt, vt, cbind, ExecType.MR);
                appG.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
                appG.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
                // group
                Group group1 = new Group(appG, Group.OperationTypes.Sort, DataType.MATRIX, vt);
                group1.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
                group1.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
                // aggregate
                Aggregate agg1 = new Aggregate(group1, Aggregate.OperationTypes.Sum, DataType.MATRIX, vt, ExecType.MR);
                agg1.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
                agg1.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
                ret = agg1;
                break;
            }
        default:
            throw new HopsException("Invalid MR append method: " + am);
    }
    return ret;
}
Also used : AppendG(org.apache.sysml.lops.AppendG) Group(org.apache.sysml.lops.Group) AppendR(org.apache.sysml.lops.AppendR) ExecType(org.apache.sysml.lops.LopProperties.ExecType) Lop(org.apache.sysml.lops.Lop) PartialAggregate(org.apache.sysml.lops.PartialAggregate) Aggregate(org.apache.sysml.lops.Aggregate) DataPartition(org.apache.sysml.lops.DataPartition) AppendM(org.apache.sysml.lops.AppendM)

Example 4 with AppendG

use of org.apache.sysml.lops.AppendG in project systemml by apache.

the class BinaryOp method constructSPAppendLop.

public static Lop constructSPAppendLop(Hop left, Hop right, DataType dt, ValueType vt, boolean cbind, Hop current) {
    Lop ret = null;
    // offset 1st input
    Lop offset = createOffsetLop(left, cbind);
    AppendMethod am = optFindAppendSPMethod(left.getDim1(), left.getDim2(), right.getDim1(), right.getDim2(), right.getRowsInBlock(), right.getColsInBlock(), right.getNnz(), cbind, dt);
    switch(am) {
        case // special case map-only append
        MR_MAPPEND:
            {
                ret = new AppendM(left.constructLops(), right.constructLops(), offset, current.getDataType(), current.getValueType(), cbind, false, ExecType.SPARK);
                break;
            }
        case // special case reduce append w/ one column block
        MR_RAPPEND:
            {
                ret = new AppendR(left.constructLops(), right.constructLops(), current.getDataType(), current.getValueType(), cbind, ExecType.SPARK);
                break;
            }
        case MR_GAPPEND:
            {
                // offset second input
                Lop offset2 = createOffsetLop(right, cbind);
                ret = new AppendG(left.constructLops(), right.constructLops(), offset, offset2, current.getDataType(), current.getValueType(), cbind, ExecType.SPARK);
                break;
            }
        case SP_GAlignedAppend:
            {
                ret = new AppendGAlignedSP(left.constructLops(), right.constructLops(), offset, current.getDataType(), current.getValueType(), cbind);
                break;
            }
        default:
            throw new HopsException("Invalid SP append method: " + am);
    }
    ret.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
    return ret;
}
Also used : AppendG(org.apache.sysml.lops.AppendG) AppendR(org.apache.sysml.lops.AppendR) Lop(org.apache.sysml.lops.Lop) AppendGAlignedSP(org.apache.sysml.lops.AppendGAlignedSP) AppendM(org.apache.sysml.lops.AppendM)

Aggregations

AppendG (org.apache.sysml.lops.AppendG)4 AppendM (org.apache.sysml.lops.AppendM)4 AppendR (org.apache.sysml.lops.AppendR)4 Lop (org.apache.sysml.lops.Lop)4 Aggregate (org.apache.sysml.lops.Aggregate)2 AppendGAlignedSP (org.apache.sysml.lops.AppendGAlignedSP)2 DataPartition (org.apache.sysml.lops.DataPartition)2 Group (org.apache.sysml.lops.Group)2 ExecType (org.apache.sysml.lops.LopProperties.ExecType)2 PartialAggregate (org.apache.sysml.lops.PartialAggregate)2
About Me
UseOf

Java Tutorials :

Simple Java RabbitMQ Example with Spring
Simple Springboot JPA Eclipeslink Example
Simple SpringBoot JPA Hibernate Example
Jackson JSON @JsonIgnore and @JsonIgnoreProperties Examples
Java Infinite recursion (StackOverflowError) Jackson solutions
Simple Java Jackson Serialize Objects to JSON and convert them back To Object
ElasticSearch 5 - Upload files using Java API in binary field Example
Java JAXB marshall unmarshall Examples from String and Stream
Java 8 forEach List and Map examples
ElasticSearch 5 Java API SearchRequestBuilder examples
Java ElasticSearch 5 examples with node index and mapping - running local
Convert String to int or Integer Java 8
Java 9 in action - JShell - Ubuntu
Java - removing invalid characters from a file name
Hello world!? or Hello Tutorial