use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.
the class TaskManager method combineUpdateIndex.
/**
* Combine update index.
*/
@SuppressWarnings("rawtypes")
public void combineUpdateIndex() {
IntOpenHashSet indexSet = null;
MatrixMeta meta = null;
for (Entry<TaskId, Task> entry : runningTask.entrySet()) {
LabeledUpdateIndexBaseTask task = (LabeledUpdateIndexBaseTask) entry.getValue().getUserTask();
IntOpenHashSet taskIndexSet = task.getIndexSet();
if (taskIndexSet != null) {
if (indexSet == null) {
indexSet = taskIndexSet;
meta = task.getMatrixMeta();
} else {
indexSet.addAll(taskIndexSet);
task.setIndexSet(null);
}
}
}
if (indexSet != null && meta != null) {
int size = indexSet.size();
int[] indexArray = new int[size];
int index = 0;
IntIterator iter = indexSet.iterator();
while (iter.hasNext()) {
indexArray[index++] = iter.nextInt();
}
Arrays.sort(indexArray);
List<PartitionKey> partKeyList = WorkerContext.get().getPSAgent().getMatrixMetaManager().getPartitions(meta.getId());
Collections.sort(partKeyList);
int partNum = partKeyList.size();
int lastPos = 0;
for (int i = 0; i < partNum; i++) {
PartitionKey partKey = partKeyList.get(i);
long endCol = partKey.getEndCol();
for (int j = lastPos; j < size; j++) {
if (indexArray[j] >= endCol) {
lastPos = j;
break;
}
}
}
// Bitmap bitmap = new Bitmap();
// int max = indexArray[size - 1];
// byte [] bitIndexArray = new byte[max / 8 + 1];
// for(int i = 0; i < size; i++){
// int bitIndex = indexArray[i] >> 3;
// int bitOffset = indexArray[i] - (bitIndex << 3);
// switch(bitOffset){
// case 0:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x01);break;
// case 1:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x02);break;
// case 2:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x04);break;
// case 3:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x08);break;
// case 4:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x10);break;
// case 5:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x20);break;
// case 6:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x40);break;
// case 7:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x80);break;
// }
// }
}
}
use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.
the class SampleNeighborParam method split.
@Override
public List<PartitionGetParam> split() {
// Get matrix meta
MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
PartitionKey[] partitions = meta.getPartitionKeys();
// Split nodeIds
KeyPart[] splits = RouterUtils.split(meta, 0, nodeIds);
assert partitions.length == splits.length;
// Generate rpc params
List<PartitionGetParam> partParams = new ArrayList<>(partitions.length);
for (int i = 0; i < partitions.length; i++) {
if (splits[i] != null && splits[i].size() > 0) {
partParams.add(new PartSampleNeighborParam(matrixId, partitions[i], splits[i], count));
}
}
return partParams;
}
use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.
the class SampleNeighborWithFilterParam method split.
@Override
public List<PartitionGetParam> split() {
// Get matrix meta
MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
PartitionKey[] partitions = meta.getPartitionKeys();
// Split nodeIds
KeyValuePart[] splits = RouterUtils.split(meta, 0, nodeIds, filterWithNeighKeys);
assert partitions.length == splits.length;
// Generate node ids
List<PartitionGetParam> partParams = new ArrayList<>(partitions.length);
for (int i = 0; i < partitions.length; i++) {
if (splits[i] != null && splits[i].size() > 0) {
partParams.add(new PartSampleNeighborWithFilterParam(matrixId, partitions[i], count, sampleType, splits[i], filterWithoutNeighKeys));
}
}
return partParams;
}
use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.
the class SampleNeighborWithTypeParam method split.
@Override
public List<PartitionGetParam> split() {
// Get matrix meta
MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
PartitionKey[] partitions = meta.getPartitionKeys();
// Split nodeIds
KeyPart[] splits = RouterUtils.split(meta, 0, nodeIds);
assert partitions.length == splits.length;
// Generate node ids
List<PartitionGetParam> partParams = new ArrayList<>(partitions.length);
for (int i = 0; i < partitions.length; i++) {
if (splits[i] != null && splits[i].size() > 0)
partParams.add(new PartSampleNeighborWithTypeParam(matrixId, partitions[i], splits[i], count, sampleType));
}
return partParams;
}
use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.
the class InitHyperLogLogParam method split.
@Override
public List<PartitionUpdateParam> split() {
MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
PartitionKey[] parts = matrixMeta.getPartitionKeys();
KeyPart[] keyParts = RouterUtils.split(matrixMeta, 0, nodes);
List<PartitionUpdateParam> params = new ArrayList<>(parts.length);
for (int i = 0; i < parts.length; i++) {
if (keyParts[i] != null && keyParts[i].size() > 0) {
params.add(new InitHyperLogLogPartParam(matrixId, parts[i], keyParts[i], p, sp, seed));
}
}
return params;
}
Aggregations