use of com.tencent.angel.ml.math2.storage.LongIntVectorStorage in project angel by Tencent.
the class SimpleBinaryOutNonZAExecutor method apply.
public static Vector apply(LongIntVector v1, LongDummyVector v2, Binary op) {
LongIntVectorStorage newStorage = (LongIntVectorStorage) StorageSwitch.apply(v1, v2, op);
if (v1.isSparse()) {
long[] v2Indices = v2.getIndices();
if (((v1.size() + v2.size()) * Constant.intersectionCoeff > Constant.sparseDenseStorageThreshold * v1.getDim())) {
int[] resValues = newStorage.getValues();
ObjectIterator<Long2IntMap.Entry> iter = v1.getStorage().entryIterator();
while (iter.hasNext()) {
Long2IntMap.Entry entry = iter.next();
newStorage.set(entry.getLongKey(), entry.getIntValue());
}
for (long idx : v2Indices) {
newStorage.set(idx, op.apply(v1.get(idx), 1));
}
} else {
// to avoid multi-rehash
int capacity = 1 << (32 - Integer.numberOfLeadingZeros((int) (v1.size() / 0.75)));
if (v1.size() + v2.size() < 1.5 * capacity) {
long size = v2.size();
for (int i = 0; i < size; i++) {
long idx = v2Indices[i];
newStorage.set(idx, op.apply(v1.get(idx), 1));
}
} else {
ObjectIterator<Long2IntMap.Entry> iter1 = v1.getStorage().entryIterator();
while (iter1.hasNext()) {
Long2IntMap.Entry entry = iter1.next();
long idx = entry.getLongKey();
newStorage.set(idx, entry.getIntValue());
}
long size = v2.size();
for (int i = 0; i < size; i++) {
long idx = v2Indices[i];
newStorage.set(idx, op.apply(v1.get(idx), 1));
}
}
}
} else {
// sorted
long[] v1Indices = v1.getStorage().getIndices();
long[] v2Indices = v2.getIndices();
if (!op.isKeepStorage() && ((v1.size() + v2.size()) * Constant.intersectionCoeff > Constant.sortedDenseStorageThreshold * v1.getDim())) {
int[] v1Values = v1.getStorage().getValues();
long size = v1.size();
for (int i = 0; i < size; i++) {
newStorage.set(v1Indices[i], v1Values[i]);
}
size = v2.size();
for (int i = 0; i < size; i++) {
long idx = v2Indices[i];
newStorage.set(idx, op.apply(newStorage.get(idx), 1));
}
} else {
int v1Pointor = 0;
int v2Pointor = 0;
long size1 = v1.size();
long size2 = v2.size();
int[] v1Values = v1.getStorage().getValues();
while (v1Pointor < size1 || v2Pointor < size2) {
if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] == v2Indices[v2Pointor]) {
newStorage.set(v1Indices[v1Pointor], op.apply(v1Values[v1Pointor], 1));
v1Pointor++;
v2Pointor++;
} else if ((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] < v2Indices[v2Pointor] || (v1Pointor < size1 && v2Pointor >= size2)) {
newStorage.set(v1Indices[v1Pointor], v1Values[v1Pointor]);
v1Pointor++;
} else if (((v1Pointor < size1 && v2Pointor < size2) && v1Indices[v1Pointor] >= v2Indices[v2Pointor]) || (v1Pointor >= size1 && v2Pointor < size2)) {
newStorage.set(v2Indices[v2Pointor], op.apply(0, 1));
v2Pointor++;
}
}
}
}
return new LongIntVector(v1.getMatrixId(), v1.getRowId(), v1.getClock(), v1.getDim(), newStorage);
}
use of com.tencent.angel.ml.math2.storage.LongIntVectorStorage in project angel by Tencent.
the class CompLongIntRowUpdateSplit method serialize.
@Override
public void serialize(ByteBuf buf) {
super.serialize(buf);
LongIntVectorStorage storage = split.getStorage();
buf.writeInt(storage.size());
if (storage instanceof LongIntSparseVectorStorage) {
ObjectIterator<Long2IntMap.Entry> iter = storage.entryIterator();
Long2IntMap.Entry entry;
while (iter.hasNext()) {
entry = iter.next();
buf.writeLong(entry.getLongKey());
buf.writeInt(entry.getIntValue());
}
} else if (storage instanceof LongIntSortedVectorStorage) {
long[] indices = storage.getIndices();
int[] values = storage.getValues();
for (int i = 0; i < indices.length; i++) {
buf.writeLong(indices[i]);
buf.writeInt(values[i]);
}
} else {
throw new UnsupportedOperationException("unsupport split for storage " + storage.getClass().getName());
}
}
use of com.tencent.angel.ml.math2.storage.LongIntVectorStorage in project angel by Tencent.
the class RangeRouterUtils method splitLongIntVector.
public static KeyValuePart[] splitLongIntVector(MatrixMeta matrixMeta, LongIntVector vector) {
LongIntVectorStorage storage = vector.getStorage();
if (storage.isSparse()) {
// Get keys and values
LongIntSparseVectorStorage sparseStorage = (LongIntSparseVectorStorage) storage;
long[] keys = sparseStorage.getIndices();
int[] values = sparseStorage.getValues();
return split(matrixMeta, vector.getRowId(), keys, values, false);
} else {
// Key and value array pair
LongIntSortedVectorStorage sortStorage = (LongIntSortedVectorStorage) storage;
long[] keys = sortStorage.getIndices();
int[] values = sortStorage.getValues();
return split(matrixMeta, vector.getRowId(), keys, values, true);
}
}
use of com.tencent.angel.ml.math2.storage.LongIntVectorStorage in project angel by Tencent.
the class HashRouterUtils method splitLongIntVector.
public static void splitLongIntVector(KeyHash hasher, MatrixMeta matrixMeta, LongIntVector vector, KeyValuePart[] dataParts) {
int dataPartNum = dataParts.length;
int dataPartNumMinus1 = dataPartNum - 1;
if (isPow2(dataPartNum)) {
LongIntVectorStorage storage = vector.getStorage();
if (storage.isSparse()) {
// Use iterator
LongIntSparseVectorStorage sparseStorage = (LongIntSparseVectorStorage) storage;
ObjectIterator<Long2IntMap.Entry> iter = sparseStorage.entryIterator();
while (iter.hasNext()) {
Long2IntMap.Entry keyValue = iter.next();
int partId = computeHashCode(hasher, keyValue.getLongKey()) & dataPartNumMinus1;
((HashLongKeysIntValuesPart) dataParts[partId]).add(keyValue.getLongKey(), keyValue.getIntValue());
}
} else {
// Key and value array pair
LongIntSortedVectorStorage sortStorage = (LongIntSortedVectorStorage) storage;
long[] keys = sortStorage.getIndices();
int[] values = sortStorage.getValues();
for (int i = 0; i < keys.length; i++) {
int partId = computeHashCode(hasher, keys[i]) & dataPartNumMinus1;
((HashLongKeysIntValuesPart) dataParts[partId]).add(keys[i], values[i]);
}
}
} else {
LongIntVectorStorage storage = vector.getStorage();
if (storage.isSparse()) {
// Use iterator
LongIntSparseVectorStorage sparseStorage = (LongIntSparseVectorStorage) storage;
ObjectIterator<Long2IntMap.Entry> iter = sparseStorage.entryIterator();
while (iter.hasNext()) {
Long2IntMap.Entry keyValue = iter.next();
int partId = computeHashCode(hasher, keyValue.getLongKey()) % dataPartNum;
((HashLongKeysIntValuesPart) dataParts[partId]).add(keyValue.getLongKey(), keyValue.getIntValue());
}
} else {
// Key and value array pair
LongIntSortedVectorStorage sortStorage = (LongIntSortedVectorStorage) storage;
long[] keys = sortStorage.getIndices();
int[] values = sortStorage.getValues();
for (int i = 0; i < keys.length; i++) {
int partId = computeHashCode(hasher, keys[i]) % dataPartNum;
((HashLongKeysIntValuesPart) dataParts[partId]).add(keys[i], values[i]);
}
}
}
}
use of com.tencent.angel.ml.math2.storage.LongIntVectorStorage in project angel by Tencent.
the class ByteBufSerdeUtils method serializeLongIntVector.
// LongFloatVector
private static void serializeLongIntVector(ByteBuf out, LongIntVector vector) {
LongIntVectorStorage storage = vector.getStorage();
if (storage.isSparse()) {
serializeInt(out, SPARSE_STORAGE_TYPE);
serializeInt(out, storage.size());
ObjectIterator<Long2IntMap.Entry> iter = storage.entryIterator();
while (iter.hasNext()) {
Long2IntMap.Entry e = iter.next();
serializeLong(out, e.getLongKey());
serializeFloat(out, e.getIntValue());
}
} else if (storage.isSorted()) {
serializeInt(out, SORTED_STORAGE_TYPE);
long[] indices = vector.getStorage().getIndices();
int[] values = vector.getStorage().getValues();
serializeLongs(out, indices);
serializeInts(out, values);
} else {
throw new UnsupportedOperationException("Unsupport storage type " + vector.getStorage().getClass());
}
}
Aggregations