use of com.baidu.hugegraph.computer.core.graph.partition.PartitionStat in project hugegraph-computer by hugegraph.
the class ComputeManager method input.
public WorkerStat input() {
WorkerStat workerStat = new WorkerStat();
this.recvManager.waitReceivedAllMessages();
Map<Integer, PeekableIterator<KvEntry>> vertices = this.recvManager.vertexPartitions();
Map<Integer, PeekableIterator<KvEntry>> edges = this.recvManager.edgePartitions();
// TODO: parallel input process
for (Map.Entry<Integer, PeekableIterator<KvEntry>> entry : vertices.entrySet()) {
int partition = entry.getKey();
PeekableIterator<KvEntry> vertexIter = entry.getValue();
PeekableIterator<KvEntry> edgesIter = edges.getOrDefault(partition, PeekableIterator.emptyIterator());
FileGraphPartition part = new FileGraphPartition(this.context, this.managers, partition);
PartitionStat partitionStat = null;
ComputerException inputException = null;
try {
partitionStat = part.input(vertexIter, edgesIter);
} catch (ComputerException e) {
inputException = e;
} finally {
try {
vertexIter.close();
edgesIter.close();
} catch (Exception e) {
String message = "Failed to close vertex or edge file " + "iterator";
ComputerException closeException = new ComputerException(message, e);
if (inputException != null) {
inputException.addSuppressed(closeException);
} else {
throw closeException;
}
}
if (inputException != null) {
throw inputException;
}
}
workerStat.add(partitionStat);
this.partitions.put(partition, part);
}
return workerStat;
}
use of com.baidu.hugegraph.computer.core.graph.partition.PartitionStat in project hugegraph-computer by hugegraph.
the class FileGraphPartition method compute.
protected PartitionStat compute(WorkerContext context, int superstep) {
LOG.info("Partition {} begin compute in superstep {}", this.partition, superstep);
try {
this.beforeCompute(superstep);
} catch (IOException e) {
throw new ComputerException("Error occurred when beforeCompute at superstep %s", e, superstep);
}
long activeVertexCount;
try {
this.computation.beforeSuperstep(context);
activeVertexCount = superstep == 0 ? this.compute0(context) : this.compute1(context);
this.computation.afterSuperstep(context);
} catch (Exception e) {
throw new ComputerException("Error occurred when compute at superstep %s", e, superstep);
}
try {
this.afterCompute(superstep);
} catch (Exception e) {
throw new ComputerException("Error occurred when afterCompute at superstep %s", e, superstep);
}
LOG.info("Partition {} finish compute in superstep {}", this.partition, superstep);
return new PartitionStat(this.partition, this.vertexCount, this.edgeCount, this.vertexCount - activeVertexCount);
}
use of com.baidu.hugegraph.computer.core.graph.partition.PartitionStat in project hugegraph-computer by hugegraph.
the class FileGraphPartition method input.
protected PartitionStat input(PeekableIterator<KvEntry> vertices, PeekableIterator<KvEntry> edges) {
try {
createFile(this.vertexFile);
createFile(this.edgeFile);
BufferedFileOutput vertexOut = new BufferedFileOutput(this.vertexFile);
BufferedFileOutput edgeOut = new BufferedFileOutput(this.edgeFile);
while (vertices.hasNext()) {
KvEntry entry = vertices.next();
Pointer key = entry.key();
Pointer value = entry.value();
this.writeVertex(key, value, vertexOut);
this.writeEdges(key, edges, edgeOut);
}
vertexOut.close();
edgeOut.close();
} catch (IOException e) {
throw new ComputerException("Failed to init FileGraphPartition '%s'", e, this.partition);
}
return new PartitionStat(this.partition, this.vertexCount, this.edgeCount, 0L);
}
use of com.baidu.hugegraph.computer.core.graph.partition.PartitionStat in project hugegraph-computer by hugegraph.
the class ComputeManager method compute.
public WorkerStat compute(ComputationContext context, int superstep) {
this.sendManager.startSend(MessageType.MSG);
WorkerStat workerStat = new WorkerStat();
Map<Integer, PartitionStat> partitionStats = new HashMap<>(this.partitions.size());
// TODO: parallel compute process.
for (FileGraphPartition<M> partition : this.partitions.values()) {
PartitionStat stat = partition.compute(context, this.computation, superstep);
partitionStats.put(stat.partitionId(), stat);
}
this.sendManager.finishSend(MessageType.MSG);
// After compute and send finish signal.
Map<Integer, MessageStat> recvStats = this.recvManager.messageStats();
for (Map.Entry<Integer, PartitionStat> entry : partitionStats.entrySet()) {
PartitionStat partStat = entry.getValue();
int partitionId = partStat.partitionId();
MessageStat sendStat = this.sendManager.messageStat(partitionId);
partStat.mergeSendMessageStat(sendStat);
MessageStat recvStat = recvStats.get(partitionId);
if (recvStat != null) {
partStat.mergeRecvMessageStat(recvStat);
}
workerStat.add(partStat);
}
return workerStat;
}
Aggregations