Search in sources :

Example 21 with PartitionStat

use of com.baidu.hugegraph.computer.core.graph.partition.PartitionStat in project hugegraph-computer by hugegraph.

the class ComputeManager method input.

public WorkerStat input() {
    WorkerStat workerStat = new WorkerStat();
    this.recvManager.waitReceivedAllMessages();
    Map<Integer, PeekableIterator<KvEntry>> vertices = this.recvManager.vertexPartitions();
    Map<Integer, PeekableIterator<KvEntry>> edges = this.recvManager.edgePartitions();
    // TODO: parallel input process
    for (Map.Entry<Integer, PeekableIterator<KvEntry>> entry : vertices.entrySet()) {
        int partition = entry.getKey();
        PeekableIterator<KvEntry> vertexIter = entry.getValue();
        PeekableIterator<KvEntry> edgesIter = edges.getOrDefault(partition, PeekableIterator.emptyIterator());
        FileGraphPartition part = new FileGraphPartition(this.context, this.managers, partition);
        PartitionStat partitionStat = null;
        ComputerException inputException = null;
        try {
            partitionStat = part.input(vertexIter, edgesIter);
        } catch (ComputerException e) {
            inputException = e;
        } finally {
            try {
                vertexIter.close();
                edgesIter.close();
            } catch (Exception e) {
                String message = "Failed to close vertex or edge file " + "iterator";
                ComputerException closeException = new ComputerException(message, e);
                if (inputException != null) {
                    inputException.addSuppressed(closeException);
                } else {
                    throw closeException;
                }
            }
            if (inputException != null) {
                throw inputException;
            }
        }
        workerStat.add(partitionStat);
        this.partitions.put(partition, part);
    }
    return workerStat;
}
Also used : KvEntry(com.baidu.hugegraph.computer.core.store.entry.KvEntry) ComputerException(com.baidu.hugegraph.computer.core.common.exception.ComputerException) ComputerException(com.baidu.hugegraph.computer.core.common.exception.ComputerException) PartitionStat(com.baidu.hugegraph.computer.core.graph.partition.PartitionStat) WorkerStat(com.baidu.hugegraph.computer.core.worker.WorkerStat) PeekableIterator(com.baidu.hugegraph.computer.core.sort.flusher.PeekableIterator) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) Map(java.util.Map)

Example 22 with PartitionStat

use of com.baidu.hugegraph.computer.core.graph.partition.PartitionStat in project hugegraph-computer by hugegraph.

the class FileGraphPartition method compute.

protected PartitionStat compute(WorkerContext context, int superstep) {
    LOG.info("Partition {} begin compute in superstep {}", this.partition, superstep);
    try {
        this.beforeCompute(superstep);
    } catch (IOException e) {
        throw new ComputerException("Error occurred when beforeCompute at superstep %s", e, superstep);
    }
    long activeVertexCount;
    try {
        this.computation.beforeSuperstep(context);
        activeVertexCount = superstep == 0 ? this.compute0(context) : this.compute1(context);
        this.computation.afterSuperstep(context);
    } catch (Exception e) {
        throw new ComputerException("Error occurred when compute at superstep %s", e, superstep);
    }
    try {
        this.afterCompute(superstep);
    } catch (Exception e) {
        throw new ComputerException("Error occurred when afterCompute at superstep %s", e, superstep);
    }
    LOG.info("Partition {} finish compute in superstep {}", this.partition, superstep);
    return new PartitionStat(this.partition, this.vertexCount, this.edgeCount, this.vertexCount - activeVertexCount);
}
Also used : PartitionStat(com.baidu.hugegraph.computer.core.graph.partition.PartitionStat) IOException(java.io.IOException) ComputerException(com.baidu.hugegraph.computer.core.common.exception.ComputerException) IOException(java.io.IOException) ComputerException(com.baidu.hugegraph.computer.core.common.exception.ComputerException)

Example 23 with PartitionStat

use of com.baidu.hugegraph.computer.core.graph.partition.PartitionStat in project hugegraph-computer by hugegraph.

the class FileGraphPartition method input.

protected PartitionStat input(PeekableIterator<KvEntry> vertices, PeekableIterator<KvEntry> edges) {
    try {
        createFile(this.vertexFile);
        createFile(this.edgeFile);
        BufferedFileOutput vertexOut = new BufferedFileOutput(this.vertexFile);
        BufferedFileOutput edgeOut = new BufferedFileOutput(this.edgeFile);
        while (vertices.hasNext()) {
            KvEntry entry = vertices.next();
            Pointer key = entry.key();
            Pointer value = entry.value();
            this.writeVertex(key, value, vertexOut);
            this.writeEdges(key, edges, edgeOut);
        }
        vertexOut.close();
        edgeOut.close();
    } catch (IOException e) {
        throw new ComputerException("Failed to init FileGraphPartition '%s'", e, this.partition);
    }
    return new PartitionStat(this.partition, this.vertexCount, this.edgeCount, 0L);
}
Also used : BufferedFileOutput(com.baidu.hugegraph.computer.core.io.BufferedFileOutput) PartitionStat(com.baidu.hugegraph.computer.core.graph.partition.PartitionStat) KvEntry(com.baidu.hugegraph.computer.core.store.entry.KvEntry) Pointer(com.baidu.hugegraph.computer.core.store.entry.Pointer) IOException(java.io.IOException) ComputerException(com.baidu.hugegraph.computer.core.common.exception.ComputerException)

Example 24 with PartitionStat

use of com.baidu.hugegraph.computer.core.graph.partition.PartitionStat in project hugegraph-computer by hugegraph.

the class ComputeManager method compute.

public WorkerStat compute(ComputationContext context, int superstep) {
    this.sendManager.startSend(MessageType.MSG);
    WorkerStat workerStat = new WorkerStat();
    Map<Integer, PartitionStat> partitionStats = new HashMap<>(this.partitions.size());
    // TODO: parallel compute process.
    for (FileGraphPartition<M> partition : this.partitions.values()) {
        PartitionStat stat = partition.compute(context, this.computation, superstep);
        partitionStats.put(stat.partitionId(), stat);
    }
    this.sendManager.finishSend(MessageType.MSG);
    // After compute and send finish signal.
    Map<Integer, MessageStat> recvStats = this.recvManager.messageStats();
    for (Map.Entry<Integer, PartitionStat> entry : partitionStats.entrySet()) {
        PartitionStat partStat = entry.getValue();
        int partitionId = partStat.partitionId();
        MessageStat sendStat = this.sendManager.messageStat(partitionId);
        partStat.mergeSendMessageStat(sendStat);
        MessageStat recvStat = recvStats.get(partitionId);
        if (recvStat != null) {
            partStat.mergeRecvMessageStat(recvStat);
        }
        workerStat.add(partStat);
    }
    return workerStat;
}
Also used : HashMap(java.util.HashMap) MessageStat(com.baidu.hugegraph.computer.core.receiver.MessageStat) PartitionStat(com.baidu.hugegraph.computer.core.graph.partition.PartitionStat) WorkerStat(com.baidu.hugegraph.computer.core.worker.WorkerStat) HashMap(java.util.HashMap) Map(java.util.Map)

Aggregations

PartitionStat (com.baidu.hugegraph.computer.core.graph.partition.PartitionStat)24 Test (org.junit.Test)15 MessageStat (com.baidu.hugegraph.computer.core.receiver.MessageStat)10 WorkerStat (com.baidu.hugegraph.computer.core.worker.WorkerStat)6 ComputerException (com.baidu.hugegraph.computer.core.common.exception.ComputerException)5 IOException (java.io.IOException)3 HashMap (java.util.HashMap)3 Map (java.util.Map)3 KvEntry (com.baidu.hugegraph.computer.core.store.entry.KvEntry)2 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)2 CountDownLatch (java.util.concurrent.CountDownLatch)2 SuperstepStat (com.baidu.hugegraph.computer.core.graph.SuperstepStat)1 Edges (com.baidu.hugegraph.computer.core.graph.edge.Edges)1 Value (com.baidu.hugegraph.computer.core.graph.value.Value)1 Vertex (com.baidu.hugegraph.computer.core.graph.vertex.Vertex)1 BufferedFileOutput (com.baidu.hugegraph.computer.core.io.BufferedFileOutput)1 ComputerOutput (com.baidu.hugegraph.computer.core.output.ComputerOutput)1 PeekableIterator (com.baidu.hugegraph.computer.core.sort.flusher.PeekableIterator)1 Pointer (com.baidu.hugegraph.computer.core.store.entry.Pointer)1 Consumers (com.baidu.hugegraph.computer.core.util.Consumers)1