Examples with FastBufferedDataInputStream org.apache.sysml.runtime.util.FastBufferedDataInputStream used on opensource projects

Example 1 with FastBufferedDataInputStream

use of org.apache.sysml.runtime.util.FastBufferedDataInputStream in project incubator-systemml by apache.

the class PartitionedBlock method readExternal.

/**
 * Redirects the default java serialization via externalizable to our default
 * hadoop writable serialization for efficient broadcast deserialization.
 *
 * @param is object input
 * @throws IOException if IOException occurs
 */
public void readExternal(ObjectInput is) throws IOException {
    DataInput dis = is;
    int code = readHeader(dis);
    if (is instanceof ObjectInputStream && code == 0) {
        // Apply only for MatrixBlock at this point as a temporary workaround
        // We will generalize this code by adding UTF functionality to support Frame
        // fast deserialize of dense/sparse blocks
        ObjectInputStream ois = (ObjectInputStream) is;
        dis = new FastBufferedDataInputStream(ois);
    }
    readPayload(dis, code);
}

Example 2 with FastBufferedDataInputStream

use of org.apache.sysml.runtime.util.FastBufferedDataInputStream in project incubator-systemml by apache.

the class MatrixBlock method readDenseBlock.

private void readDenseBlock(DataInput in) throws IOException, DMLRuntimeException {
    if (// allocate block
    !allocateDenseBlock(false))
        denseBlock.reset(rlen, clen);
    DenseBlock a = getDenseBlock();
    long nnz = 0;
    if (in instanceof MatrixBlockDataInput) {
        // fast deserialize
        MatrixBlockDataInput mbin = (MatrixBlockDataInput) in;
        for (int i = 0; i < a.numBlocks(); i++) nnz += mbin.readDoubleArray(a.size(i), a.valuesAt(i));
    } else if (in instanceof DataInputBuffer && MRJobConfiguration.USE_BINARYBLOCK_SERIALIZATION) {
        // workaround because sequencefile.reader.next(key, value) does not yet support serialization framework
        DataInputBuffer din = (DataInputBuffer) in;
        try (FastBufferedDataInputStream mbin = new FastBufferedDataInputStream(din)) {
            for (int i = 0; i < a.numBlocks(); i++) nnz += mbin.readDoubleArray(a.size(i), a.valuesAt(i));
        }
    } else {
        // default deserialize
        for (int i = 0; i < rlen; i++) {
            double[] avals = a.values(i);
            int aix = a.pos(i);
            for (int j = 0; j < clen; j++) nnz += ((avals[aix + j] = in.readDouble()) != 0) ? 1 : 0;
        }
    }
    nonZeros = nnz;
}

Example 3 with FastBufferedDataInputStream

use of org.apache.sysml.runtime.util.FastBufferedDataInputStream in project incubator-systemml by apache.

the class CorrMatrixBlock method readExternal.

/**
 * Redirects the default java serialization via externalizable to our default
 * hadoop writable serialization for efficient deserialization.
 *
 * @param is object input
 * @throws IOException if IOException occurs
 */
public void readExternal(ObjectInput is) throws IOException {
    DataInput dis = is;
    if (is instanceof ObjectInputStream) {
        // fast deserialize of dense/sparse blocks
        ObjectInputStream ois = (ObjectInputStream) is;
        dis = new FastBufferedDataInputStream(ois);
    }
    readHeaderAndPayload(dis);
}

Example 4 with FastBufferedDataInputStream

use of org.apache.sysml.runtime.util.FastBufferedDataInputStream in project incubator-systemml by apache.

the class RowMatrixBlock method readExternal.

/**
 * Redirects the default java serialization via externalizable to our default
 * hadoop writable serialization for efficient deserialization.
 *
 * @param is object input
 * @throws IOException if IOException occurs
 */
public void readExternal(ObjectInput is) throws IOException {
    DataInput dis = is;
    if (is instanceof ObjectInputStream) {
        // fast deserialize of dense/sparse blocks
        ObjectInputStream ois = (ObjectInputStream) is;
        dis = new FastBufferedDataInputStream(ois);
    }
    readHeaderAndPayload(dis);
}

Example 5 with FastBufferedDataInputStream

use of org.apache.sysml.runtime.util.FastBufferedDataInputStream in project incubator-systemml by apache.

the class MatrixBlock method readExternal.

/**
 * Redirects the default java serialization via externalizable to our default
 * hadoop writable serialization for efficient broadcast/rdd deserialization.
 *
 * @param is object input
 * @throws IOException if IOException occurs
 */
public void readExternal(ObjectInput is) throws IOException {
    if (is instanceof ObjectInputStream) {
        // fast deserialize of dense/sparse blocks
        ObjectInputStream ois = (ObjectInputStream) is;
        FastBufferedDataInputStream fis = new FastBufferedDataInputStream(ois);
        readFields(fis);
    } else {
        // default deserialize (general case)
        readFields(is);
    }
}