Examples with MessageStat com.baidu.hugegraph.computer.core.receiver.MessageStat used on opensource projects

Example 16 with MessageStat

use of com.baidu.hugegraph.computer.core.receiver.MessageStat in project hugegraph-computer by hugegraph.

the class PartitionStatTest method testReadWrite.

@Test
public void testReadWrite() throws IOException {
    PartitionStat stat1 = new PartitionStat(0, 1L, 2L, 0L);
    PartitionStat stat1ReadObj = new PartitionStat();
    UnitTestBase.assertEqualAfterWriteAndRead(stat1, stat1ReadObj);
    PartitionStat stat2 = new PartitionStat(1, 4L, 3L, 2L);
    stat2.mergeSendMessageStat(new MessageStat(5L, 6L));
    stat2.mergeRecvMessageStat(new MessageStat(7L, 8L));
    PartitionStat stat2ReadObj = new PartitionStat();
    UnitTestBase.assertEqualAfterWriteAndRead(stat2, stat2ReadObj);
}

Example 17 with MessageStat

use of com.baidu.hugegraph.computer.core.receiver.MessageStat in project hugegraph-computer by hugegraph.

the class ComputeManager method compute.

public WorkerStat compute(WorkerContext context, int superstep) {
    this.sendManager.startSend(MessageType.MSG);
    WorkerStat workerStat = new WorkerStat();
    Map<Integer, PartitionStat> stats = new ConcurrentHashMap<>();
    /*
         * Remark: The main thread can perceive the partition compute exception
         * only after all partition compute completed, and only record the last
         * exception.
         */
    Consumers<FileGraphPartition> consumers = new Consumers<>(this.computeExecutor, partition -> {
        PartitionStat stat = partition.compute(context, superstep);
        stats.put(stat.partitionId(), stat);
    });
    consumers.start("partition-compute");
    try {
        for (FileGraphPartition partition : this.partitions.values()) {
            consumers.provide(partition);
        }
        consumers.await();
    } catch (Throwable t) {
        throw new ComputerException("An exception occurred when " + "partition parallel compute", t);
    }
    this.sendManager.finishSend(MessageType.MSG);
    // After compute and send finish signal.
    Map<Integer, MessageStat> recvStats = this.recvManager.messageStats();
    for (Map.Entry<Integer, PartitionStat> entry : stats.entrySet()) {
        PartitionStat partStat = entry.getValue();
        int partitionId = partStat.partitionId();
        MessageStat sendStat = this.sendManager.messageStat(partitionId);
        partStat.mergeSendMessageStat(sendStat);
        MessageStat recvStat = recvStats.get(partitionId);
        if (recvStat != null) {
            partStat.mergeRecvMessageStat(recvStat);
        }
        workerStat.add(partStat);
    }
    return workerStat;
}

Example 18 with MessageStat

use of com.baidu.hugegraph.computer.core.receiver.MessageStat in project hugegraph-computer by hugegraph.

the class ComputeManager method compute.

public WorkerStat compute(ComputationContext context, int superstep) {
    this.sendManager.startSend(MessageType.MSG);
    WorkerStat workerStat = new WorkerStat();
    Map<Integer, PartitionStat> partitionStats = new HashMap<>(this.partitions.size());
    // TODO: parallel compute process.
    for (FileGraphPartition<M> partition : this.partitions.values()) {
        PartitionStat stat = partition.compute(context, this.computation, superstep);
        partitionStats.put(stat.partitionId(), stat);
    }
    this.sendManager.finishSend(MessageType.MSG);
    // After compute and send finish signal.
    Map<Integer, MessageStat> recvStats = this.recvManager.messageStats();
    for (Map.Entry<Integer, PartitionStat> entry : partitionStats.entrySet()) {
        PartitionStat partStat = entry.getValue();
        int partitionId = partStat.partitionId();
        MessageStat sendStat = this.sendManager.messageStat(partitionId);
        partStat.mergeSendMessageStat(sendStat);
        MessageStat recvStat = recvStats.get(partitionId);
        if (recvStat != null) {
            partStat.mergeRecvMessageStat(recvStat);
        }
        workerStat.add(partStat);
    }
    return workerStat;
}

Example 19 with MessageStat

use of com.baidu.hugegraph.computer.core.receiver.MessageStat in project hugegraph-computer by hugegraph.

the class MessageSendManager method sortAndSendLastBuffer.

private MessageStat sortAndSendLastBuffer(Map<Integer, WriteBuffers> all, MessageType type) {
    MessageStat messageWritten = new MessageStat();
    List<Future<?>> futures = new ArrayList<>(all.size());
    // Sort and send the last buffer
    for (Map.Entry<Integer, WriteBuffers> entry : all.entrySet()) {
        int partitionId = entry.getKey();
        WriteBuffers buffer = entry.getValue();
        /*
             * If the last buffer has already been sorted and sent (empty),
             * there is no need to send again here
             */
        if (!buffer.isEmpty()) {
            buffer.prepareSorting();
            futures.add(this.sortThenSend(partitionId, type, buffer));
        }
        // Record total message count & bytes
        messageWritten.increase(buffer.messageWritten());
    }
    this.checkException();
    // Wait all future finished
    try {
        for (Future<?> future : futures) {
            future.get(Constants.FUTURE_TIMEOUT, TimeUnit.SECONDS);
        }
    } catch (TimeoutException e) {
        throw new ComputerException("Timed out to wait for sorting task " + "to finished", e);
    } catch (InterruptedException | ExecutionException e) {
        throw new ComputerException("Failed to wait for sorting task " + "to finished", e);
    }
    return messageWritten;
}