Examples with WorkerStat com.baidu.hugegraph.computer.core.worker.WorkerStat used on opensource projects

Example 1 with WorkerStat

use of com.baidu.hugegraph.computer.core.worker.WorkerStat in project hugegraph-computer by hugegraph.

the class Bsp4Master method waitWorkersStepDone.

/**
 * Wait workers finish specified superstep. The master receives the
 * worker stat from all workers, calls algorithm's master computation,
 * check the max iteration count, and then calls masterSuperstepDone to
 * synchronize superstep result.
 */
public List<WorkerStat> waitWorkersStepDone(int superstep) {
    LOG.info("Master is waiting for workers superstep-done({})", superstep);
    String path = this.constructPath(BspEvent.BSP_WORKER_STEP_DONE, superstep);
    List<byte[]> list = this.waitOnWorkersEvent(path, this.barrierOnWorkersTimeout());
    List<WorkerStat> result = new ArrayList<>(this.workerCount());
    for (byte[] bytes : list) {
        WorkerStat workerStat = new WorkerStat();
        SerializeUtil.fromBytes(bytes, workerStat);
        result.add(workerStat);
    }
    LOG.info("Master waited workers superstep-done({}), workers stat: {}", superstep, result);
    return result;
}

Example 2 with WorkerStat

use of com.baidu.hugegraph.computer.core.worker.WorkerStat in project hugegraph-computer by hugegraph.

the class EtcdBspTest method testInput.

@Test
public void testInput() throws InterruptedException {
    // If both two threads reach countDown, it means no exception is thrown.
    WorkerStat workerStat = new WorkerStat();
    workerStat.add(new PartitionStat(0, 100L, 200L, 0L));
    workerStat.add(new PartitionStat(1, 200L, 300L, 0L));
    CountDownLatch countDownLatch = new CountDownLatch(2);
    this.executorService.submit(() -> {
        this.bsp4Master.masterResumeDone(-1);
        this.bsp4Master.waitWorkersInputDone();
        this.bsp4Master.masterInputDone();
        List<WorkerStat> workerStats = this.bsp4Master.waitWorkersStepDone(-1);
        Assert.assertEquals(1, workerStats.size());
        Assert.assertEquals(workerStat, workerStats.get(0));
        countDownLatch.countDown();
    });
    this.executorService.submit(() -> {
        int firstSuperStep = this.bsp4Worker.waitMasterResumeDone();
        Assert.assertEquals(-1, firstSuperStep);
        this.bsp4Worker.workerInputDone();
        this.bsp4Worker.waitMasterInputDone();
        this.bsp4Worker.workerStepDone(-1, workerStat);
        countDownLatch.countDown();
    });
    countDownLatch.await();
}

Example 3 with WorkerStat

use of com.baidu.hugegraph.computer.core.worker.WorkerStat in project hugegraph-computer by hugegraph.

the class EtcdBspTest method testIterate.

@Test
public void testIterate() throws InterruptedException {
    // If both two threads reach countDown, it means no exception is thrown.
    WorkerStat workerStat = new WorkerStat();
    workerStat.add(new PartitionStat(0, 100L, 200L, 0L));
    workerStat.add(new PartitionStat(1, 200L, 300L, 0L));
    CountDownLatch countDownLatch = new CountDownLatch(2);
    this.executorService.submit(() -> {
        for (int i = 0; i < this.maxSuperStep; i++) {
            this.bsp4Master.waitWorkersStepPrepareDone(i);
            this.bsp4Master.masterStepPrepareDone(i);
            this.bsp4Master.waitWorkersStepComputeDone(i);
            this.bsp4Master.masterStepComputeDone(i);
            List<WorkerStat> list = this.bsp4Master.waitWorkersStepDone(i);
            SuperstepStat superstepStat = new SuperstepStat();
            for (WorkerStat workerStat1 : list) {
                superstepStat.increase(workerStat1);
            }
            if (i == this.maxSuperStep - 1) {
                superstepStat.inactivate();
            }
            this.bsp4Master.masterStepDone(i, superstepStat);
        }
        countDownLatch.countDown();
    });
    this.executorService.submit(() -> {
        int superstep = -1;
        SuperstepStat superstepStat = null;
        while (superstepStat == null || superstepStat.active()) {
            superstep++;
            this.bsp4Worker.workerStepPrepareDone(superstep);
            this.bsp4Worker.waitMasterStepPrepareDone(superstep);
            this.bsp4Worker.workerStepComputeDone(superstep);
            this.bsp4Worker.waitMasterStepComputeDone(superstep);
            PartitionStat stat1 = new PartitionStat(0, 100L, 200L, 50L);
            PartitionStat stat2 = new PartitionStat(1, 200L, 300L, 80L);
            WorkerStat workerStatInSuperstep = new WorkerStat();
            workerStatInSuperstep.add(stat1);
            workerStatInSuperstep.add(stat2);
            // Sleep some time to simulate the worker do computation.
            UnitTestBase.sleep(100L);
            this.bsp4Worker.workerStepDone(superstep, workerStatInSuperstep);
            superstepStat = this.bsp4Worker.waitMasterStepDone(superstep);
        }
        countDownLatch.countDown();
    });
    countDownLatch.await();
}

Example 4 with WorkerStat

use of com.baidu.hugegraph.computer.core.worker.WorkerStat in project hugegraph-computer by hugegraph.

the class SuperstepStatTest method testIncreaseWorkerStat.

@Test
public void testIncreaseWorkerStat() {
    SuperstepStat stat = new SuperstepStat();
    PartitionStat partitionStat1 = new PartitionStat(1, 4L, 3L, 2L);
    partitionStat1.mergeSendMessageStat(new MessageStat(5L, 6L));
    partitionStat1.mergeRecvMessageStat(new MessageStat(7L, 8L));
    PartitionStat partitionStat2 = new PartitionStat(2, 14L, 13L, 12L);
    partitionStat2.mergeSendMessageStat(new MessageStat(15L, 16L));
    partitionStat2.mergeRecvMessageStat(new MessageStat(17L, 18L));
    WorkerStat workerStat = new WorkerStat();
    workerStat.add(partitionStat1);
    workerStat.add(partitionStat2);
    stat.increase(workerStat);
    Assert.assertEquals(18, stat.vertexCount());
    Assert.assertEquals(16, stat.edgeCount());
    Assert.assertEquals(14L, stat.finishedVertexCount());
    Assert.assertEquals(20L, stat.messageSendCount());
    Assert.assertEquals(22L, stat.messageSendBytes());
    Assert.assertEquals(24L, stat.messageRecvCount());
    Assert.assertEquals(26L, stat.messageRecvBytes());
}

Example 5 with WorkerStat

use of com.baidu.hugegraph.computer.core.worker.WorkerStat in project hugegraph-computer by hugegraph.

the class MasterService method execute.

/**
 * Execute the graph. First determines which superstep to start from. And
 * then execute the superstep iteration.
 * After the superstep iteration, output the result.
 */
public void execute() {
    StopWatch watcher = new StopWatch();
    this.checkInited();
    LOG.info("{} MasterService execute", this);
    /*
         * Step 1: Determines which superstep to start from, and resume this
         * superstep.
         */
    int superstep = this.superstepToResume();
    LOG.info("{} MasterService resume from superstep: {}", this, superstep);
    /*
         * TODO: Get input splits from HugeGraph if resume from
         * Constants.INPUT_SUPERSTEP.
         */
    this.bsp4Master.masterResumeDone(superstep);
    /*
         * Step 2: Input superstep for loading vertices and edges.
         * This step may be skipped if resume from other superstep than
         * Constants.INPUT_SUPERSTEP.
         */
    SuperstepStat superstepStat;
    watcher.start();
    if (superstep == Constants.INPUT_SUPERSTEP) {
        superstepStat = this.inputstep();
        superstep++;
    } else {
        // TODO: Get superstepStat from bsp service.
        superstepStat = null;
    }
    watcher.stop();
    LOG.info("{} MasterService input step cost: {}", this, TimeUtil.readableTime(watcher.getTime()));
    E.checkState(superstep <= this.maxSuperStep, "The superstep {} can't be > maxSuperStep {}", superstep, this.maxSuperStep);
    watcher.reset();
    watcher.start();
    // Step 3: Iteration computation of all supersteps.
    for (; superstepStat.active(); superstep++) {
        LOG.info("{} MasterService superstep {} started", this, superstep);
        /*
             * Superstep iteration. The steps in each superstep are:
             * 1) Master waits workers superstep prepared.
             * 2) All managers call beforeSuperstep.
             * 3) Master signals the workers that the master prepared
             *    superstep.
             * 4) Master waits the workers do vertex computation.
             * 5) Master signal the workers that all workers have finished
             *    vertex computation.
             * 6) Master waits the workers end the superstep, and get
             *    superstepStat.
             * 7) Master compute whether to continue the next superstep
             *    iteration.
             * 8) All managers call afterSuperstep.
             * 9) Master signals the workers with superstepStat, and workers
             *    know whether to continue the next superstep iteration.
             */
        this.bsp4Master.waitWorkersStepPrepareDone(superstep);
        this.managers.beforeSuperstep(this.config, superstep);
        this.bsp4Master.masterStepPrepareDone(superstep);
        this.bsp4Master.waitWorkersStepComputeDone(superstep);
        this.bsp4Master.masterStepComputeDone(superstep);
        List<WorkerStat> workerStats = this.bsp4Master.waitWorkersStepDone(superstep);
        superstepStat = SuperstepStat.from(workerStats);
        SuperstepContext context = new SuperstepContext(superstep, superstepStat);
        // Call master compute(), note the worker afterSuperstep() is done
        boolean masterContinue = this.masterComputation.compute(context);
        if (this.finishedIteration(masterContinue, context)) {
            superstepStat.inactivate();
        }
        this.managers.afterSuperstep(this.config, superstep);
        this.bsp4Master.masterStepDone(superstep, superstepStat);
        LOG.info("{} MasterService superstep {} finished", this, superstep);
    }
    watcher.stop();
    LOG.info("{} MasterService compute step cost: {}", this, TimeUtil.readableTime(watcher.getTime()));
    watcher.reset();
    watcher.start();
    // Step 4: Output superstep for outputting results.
    this.outputstep();
    watcher.stop();
    LOG.info("{} MasterService output step cost: {}", this, TimeUtil.readableTime(watcher.getTime()));
}