Examples with WorkerStat com.baidu.hugegraph.computer.core.worker.WorkerStat used on opensource projects

Example 6 with WorkerStat

use of com.baidu.hugegraph.computer.core.worker.WorkerStat in project hugegraph-computer by hugegraph.

the class MasterService method inputstep.

/**
 * Coordinate with workers to load vertices and edges from HugeGraph. There
 * are two phases in inputstep. First phase is get input splits from
 * master, and read the vertices and edges from input splits. The second
 * phase is after all workers read input splits, the workers merge the
 * vertices and edges to get the stats for each partition.
 */
private SuperstepStat inputstep() {
    LOG.info("{} MasterService inputstep started", this);
    this.bsp4Master.waitWorkersInputDone();
    this.bsp4Master.masterInputDone();
    List<WorkerStat> workerStats = this.bsp4Master.waitWorkersStepDone(Constants.INPUT_SUPERSTEP);
    SuperstepStat superstepStat = SuperstepStat.from(workerStats);
    this.bsp4Master.masterStepDone(Constants.INPUT_SUPERSTEP, superstepStat);
    LOG.info("{} MasterService inputstep finished with superstat {}", this, superstepStat);
    return superstepStat;
}

Example 7 with WorkerStat

use of com.baidu.hugegraph.computer.core.worker.WorkerStat in project hugegraph-computer by hugegraph.

the class ComputeManager method compute.

public WorkerStat compute(WorkerContext context, int superstep) {
    this.sendManager.startSend(MessageType.MSG);
    WorkerStat workerStat = new WorkerStat();
    Map<Integer, PartitionStat> stats = new ConcurrentHashMap<>();
    /*
         * Remark: The main thread can perceive the partition compute exception
         * only after all partition compute completed, and only record the last
         * exception.
         */
    Consumers<FileGraphPartition> consumers = new Consumers<>(this.computeExecutor, partition -> {
        PartitionStat stat = partition.compute(context, superstep);
        stats.put(stat.partitionId(), stat);
    });
    consumers.start("partition-compute");
    try {
        for (FileGraphPartition partition : this.partitions.values()) {
            consumers.provide(partition);
        }
        consumers.await();
    } catch (Throwable t) {
        throw new ComputerException("An exception occurred when " + "partition parallel compute", t);
    }
    this.sendManager.finishSend(MessageType.MSG);
    // After compute and send finish signal.
    Map<Integer, MessageStat> recvStats = this.recvManager.messageStats();
    for (Map.Entry<Integer, PartitionStat> entry : stats.entrySet()) {
        PartitionStat partStat = entry.getValue();
        int partitionId = partStat.partitionId();
        MessageStat sendStat = this.sendManager.messageStat(partitionId);
        partStat.mergeSendMessageStat(sendStat);
        MessageStat recvStat = recvStats.get(partitionId);
        if (recvStat != null) {
            partStat.mergeRecvMessageStat(recvStat);
        }
        workerStat.add(partStat);
    }
    return workerStat;
}

Example 8 with WorkerStat

use of com.baidu.hugegraph.computer.core.worker.WorkerStat in project hugegraph-computer by hugegraph.

the class ComputeManager method input.

public WorkerStat input() {
    WorkerStat workerStat = new WorkerStat();
    this.recvManager.waitReceivedAllMessages();
    Map<Integer, PeekableIterator<KvEntry>> vertices = this.recvManager.vertexPartitions();
    Map<Integer, PeekableIterator<KvEntry>> edges = this.recvManager.edgePartitions();
    // TODO: parallel input process
    for (Map.Entry<Integer, PeekableIterator<KvEntry>> entry : vertices.entrySet()) {
        int partition = entry.getKey();
        PeekableIterator<KvEntry> vertexIter = entry.getValue();
        PeekableIterator<KvEntry> edgesIter = edges.getOrDefault(partition, PeekableIterator.emptyIterator());
        FileGraphPartition part = new FileGraphPartition(this.context, this.managers, partition);
        PartitionStat partitionStat = null;
        ComputerException inputException = null;
        try {
            partitionStat = part.input(vertexIter, edgesIter);
        } catch (ComputerException e) {
            inputException = e;
        } finally {
            try {
                vertexIter.close();
                edgesIter.close();
            } catch (Exception e) {
                String message = "Failed to close vertex or edge file " + "iterator";
                ComputerException closeException = new ComputerException(message, e);
                if (inputException != null) {
                    inputException.addSuppressed(closeException);
                } else {
                    throw closeException;
                }
            }
            if (inputException != null) {
                throw inputException;
            }
        }
        workerStat.add(partitionStat);
        this.partitions.put(partition, part);
    }
    return workerStat;
}

Example 9 with WorkerStat

use of com.baidu.hugegraph.computer.core.worker.WorkerStat in project hugegraph-computer by hugegraph.

the class ComputeManager method compute.

public WorkerStat compute(ComputationContext context, int superstep) {
    this.sendManager.startSend(MessageType.MSG);
    WorkerStat workerStat = new WorkerStat();
    Map<Integer, PartitionStat> partitionStats = new HashMap<>(this.partitions.size());
    // TODO: parallel compute process.
    for (FileGraphPartition<M> partition : this.partitions.values()) {
        PartitionStat stat = partition.compute(context, this.computation, superstep);
        partitionStats.put(stat.partitionId(), stat);
    }
    this.sendManager.finishSend(MessageType.MSG);
    // After compute and send finish signal.
    Map<Integer, MessageStat> recvStats = this.recvManager.messageStats();
    for (Map.Entry<Integer, PartitionStat> entry : partitionStats.entrySet()) {
        PartitionStat partStat = entry.getValue();
        int partitionId = partStat.partitionId();
        MessageStat sendStat = this.sendManager.messageStat(partitionId);
        partStat.mergeSendMessageStat(sendStat);
        MessageStat recvStat = recvStats.get(partitionId);
        if (recvStat != null) {
            partStat.mergeRecvMessageStat(recvStat);
        }
        workerStat.add(partStat);
    }
    return workerStat;
}