Examples with SuperstepStat com.baidu.hugegraph.computer.core.graph.SuperstepStat used on opensource projects

Example 1 with SuperstepStat

use of com.baidu.hugegraph.computer.core.graph.SuperstepStat in project hugegraph-computer by hugegraph.

the class Bsp4Worker method waitMasterStepDone.

/**
 * The master set this signal after all workers signaled superstepDone,
 * and master computes MasterComputation, and broadcast all aggregators to
 * works.
 */
public SuperstepStat waitMasterStepDone(int superstep) {
    LOG.info("Worker({}) is waiting for master superstep-done({})", this.workerInfo.id(), superstep);
    String path = this.constructPath(BspEvent.BSP_MASTER_STEP_DONE, superstep);
    byte[] bytes = this.bspClient().get(path, this.barrierOnMasterTimeout(), this.logInterval());
    SuperstepStat superstepStat = new SuperstepStat();
    SerializeUtil.fromBytes(bytes, superstepStat);
    LOG.info("Worker({}) waited master superstep-done({}), graph stat: {}", this.workerInfo.id(), superstep, superstepStat);
    return superstepStat;
}

Example 2 with SuperstepStat

use of com.baidu.hugegraph.computer.core.graph.SuperstepStat in project hugegraph-computer by hugegraph.

the class WorkerService method inputstep.

/**
 * Load vertices and edges from HugeGraph. There are two phases in
 * inputstep. First phase is get input splits from master, and read the
 * vertices and edges from input splits. The second phase is after all
 * workers read input splits, the workers merge the vertices and edges to
 * get the stats for each partition.
 */
private SuperstepStat inputstep() {
    LOG.info("{} WorkerService inputstep started", this);
    WorkerInputManager manager = this.managers.get(WorkerInputManager.NAME);
    manager.loadGraph();
    this.bsp4Worker.workerInputDone();
    this.bsp4Worker.waitMasterInputDone();
    WorkerStat workerStat = this.computeManager.input();
    this.bsp4Worker.workerStepDone(Constants.INPUT_SUPERSTEP, workerStat);
    SuperstepStat superstepStat = this.bsp4Worker.waitMasterStepDone(Constants.INPUT_SUPERSTEP);
    manager.close(this.config);
    LOG.info("{} WorkerService inputstep finished", this);
    return superstepStat;
}

Example 3 with SuperstepStat

use of com.baidu.hugegraph.computer.core.graph.SuperstepStat in project hugegraph-computer by hugegraph.

the class WorkerService method execute.

/**
 * Execute the superstep in worker. It first wait master witch superstep
 * to start from. And then do the superstep iteration until master's
 * superstepStat is inactive.
 */
public void execute() {
    this.checkInited();
    LOG.info("{} WorkerService execute", this);
    // TODO: determine superstep if fail over is enabled.
    int superstep = this.bsp4Worker.waitMasterResumeDone();
    SuperstepStat superstepStat;
    if (superstep == Constants.INPUT_SUPERSTEP) {
        superstepStat = this.inputstep();
        superstep++;
    } else {
        // TODO: Get superstepStat from bsp service.
        superstepStat = null;
    }
    /*
         * The master determine whether to execute the next superstep. The
         * superstepStat is active while master decides to execute the next
         * superstep.
         */
    while (superstepStat.active()) {
        WorkerContext context = new SuperstepContext(superstep, superstepStat);
        LOG.info("Start computation of superstep {}", superstep);
        if (superstep > 0) {
            this.computeManager.takeRecvedMessages();
        }
        /*
             * Call beforeSuperstep() before all workers compute() called.
             *
             * NOTE: keep computeManager.compute() called after
             * managers.beforeSuperstep().
             */
        this.managers.beforeSuperstep(this.config, superstep);
        /*
             * Notify master by each worker, when the master received all
             * workers signal, then notify all workers to do compute().
             */
        this.bsp4Worker.workerStepPrepareDone(superstep);
        this.bsp4Worker.waitMasterStepPrepareDone(superstep);
        WorkerStat workerStat = this.computeManager.compute(context, superstep);
        this.bsp4Worker.workerStepComputeDone(superstep);
        this.bsp4Worker.waitMasterStepComputeDone(superstep);
        /*
             * Call afterSuperstep() after all workers compute() is done.
             *
             * NOTE: keep managers.afterSuperstep() called after
             * computeManager.compute(), because managers may rely on
             * computation, like WorkerAggrManager send aggregators to master
             * after called aggregateValue(String name, V value) in computation.
             */
        this.managers.afterSuperstep(this.config, superstep);
        this.bsp4Worker.workerStepDone(superstep, workerStat);
        LOG.info("End computation of superstep {}", superstep);
        superstepStat = this.bsp4Worker.waitMasterStepDone(superstep);
        superstep++;
    }
    this.outputstep();
}

Example 4 with SuperstepStat

use of com.baidu.hugegraph.computer.core.graph.SuperstepStat in project hugegraph-computer by hugegraph.

the class EtcdBspTest method testIterate.

@Test
public void testIterate() throws InterruptedException {
    // If both two threads reach countDown, it means no exception is thrown.
    WorkerStat workerStat = new WorkerStat();
    workerStat.add(new PartitionStat(0, 100L, 200L, 0L));
    workerStat.add(new PartitionStat(1, 200L, 300L, 0L));
    CountDownLatch countDownLatch = new CountDownLatch(2);
    this.executorService.submit(() -> {
        for (int i = 0; i < this.maxSuperStep; i++) {
            this.bsp4Master.waitWorkersStepPrepareDone(i);
            this.bsp4Master.masterStepPrepareDone(i);
            this.bsp4Master.waitWorkersStepComputeDone(i);
            this.bsp4Master.masterStepComputeDone(i);
            List<WorkerStat> list = this.bsp4Master.waitWorkersStepDone(i);
            SuperstepStat superstepStat = new SuperstepStat();
            for (WorkerStat workerStat1 : list) {
                superstepStat.increase(workerStat1);
            }
            if (i == this.maxSuperStep - 1) {
                superstepStat.inactivate();
            }
            this.bsp4Master.masterStepDone(i, superstepStat);
        }
        countDownLatch.countDown();
    });
    this.executorService.submit(() -> {
        int superstep = -1;
        SuperstepStat superstepStat = null;
        while (superstepStat == null || superstepStat.active()) {
            superstep++;
            this.bsp4Worker.workerStepPrepareDone(superstep);
            this.bsp4Worker.waitMasterStepPrepareDone(superstep);
            this.bsp4Worker.workerStepComputeDone(superstep);
            this.bsp4Worker.waitMasterStepComputeDone(superstep);
            PartitionStat stat1 = new PartitionStat(0, 100L, 200L, 50L);
            PartitionStat stat2 = new PartitionStat(1, 200L, 300L, 80L);
            WorkerStat workerStatInSuperstep = new WorkerStat();
            workerStatInSuperstep.add(stat1);
            workerStatInSuperstep.add(stat2);
            // Sleep some time to simulate the worker do computation.
            UnitTestBase.sleep(100L);
            this.bsp4Worker.workerStepDone(superstep, workerStatInSuperstep);
            superstepStat = this.bsp4Worker.waitMasterStepDone(superstep);
        }
        countDownLatch.countDown();
    });
    countDownLatch.await();
}

Example 5 with SuperstepStat

use of com.baidu.hugegraph.computer.core.graph.SuperstepStat in project hugegraph-computer by hugegraph.

the class MasterService method execute.

/**
 * Execute the graph. First determines which superstep to start from. And
 * then execute the superstep iteration.
 * After the superstep iteration, output the result.
 */
public void execute() {
    StopWatch watcher = new StopWatch();
    this.checkInited();
    LOG.info("{} MasterService execute", this);
    /*
         * Step 1: Determines which superstep to start from, and resume this
         * superstep.
         */
    int superstep = this.superstepToResume();
    LOG.info("{} MasterService resume from superstep: {}", this, superstep);
    /*
         * TODO: Get input splits from HugeGraph if resume from
         * Constants.INPUT_SUPERSTEP.
         */
    this.bsp4Master.masterResumeDone(superstep);
    /*
         * Step 2: Input superstep for loading vertices and edges.
         * This step may be skipped if resume from other superstep than
         * Constants.INPUT_SUPERSTEP.
         */
    SuperstepStat superstepStat;
    watcher.start();
    if (superstep == Constants.INPUT_SUPERSTEP) {
        superstepStat = this.inputstep();
        superstep++;
    } else {
        // TODO: Get superstepStat from bsp service.
        superstepStat = null;
    }
    watcher.stop();
    LOG.info("{} MasterService input step cost: {}", this, TimeUtil.readableTime(watcher.getTime()));
    E.checkState(superstep <= this.maxSuperStep, "The superstep {} can't be > maxSuperStep {}", superstep, this.maxSuperStep);
    watcher.reset();
    watcher.start();
    // Step 3: Iteration computation of all supersteps.
    for (; superstepStat.active(); superstep++) {
        LOG.info("{} MasterService superstep {} started", this, superstep);
        /*
             * Superstep iteration. The steps in each superstep are:
             * 1) Master waits workers superstep prepared.
             * 2) All managers call beforeSuperstep.
             * 3) Master signals the workers that the master prepared
             *    superstep.
             * 4) Master waits the workers do vertex computation.
             * 5) Master signal the workers that all workers have finished
             *    vertex computation.
             * 6) Master waits the workers end the superstep, and get
             *    superstepStat.
             * 7) Master compute whether to continue the next superstep
             *    iteration.
             * 8) All managers call afterSuperstep.
             * 9) Master signals the workers with superstepStat, and workers
             *    know whether to continue the next superstep iteration.
             */
        this.bsp4Master.waitWorkersStepPrepareDone(superstep);
        this.managers.beforeSuperstep(this.config, superstep);
        this.bsp4Master.masterStepPrepareDone(superstep);
        this.bsp4Master.waitWorkersStepComputeDone(superstep);
        this.bsp4Master.masterStepComputeDone(superstep);
        List<WorkerStat> workerStats = this.bsp4Master.waitWorkersStepDone(superstep);
        superstepStat = SuperstepStat.from(workerStats);
        SuperstepContext context = new SuperstepContext(superstep, superstepStat);
        // Call master compute(), note the worker afterSuperstep() is done
        boolean masterContinue = this.masterComputation.compute(context);
        if (this.finishedIteration(masterContinue, context)) {
            superstepStat.inactivate();
        }
        this.managers.afterSuperstep(this.config, superstep);
        this.bsp4Master.masterStepDone(superstep, superstepStat);
        LOG.info("{} MasterService superstep {} finished", this, superstep);
    }
    watcher.stop();
    LOG.info("{} MasterService compute step cost: {}", this, TimeUtil.readableTime(watcher.getTime()));
    watcher.reset();
    watcher.start();
    // Step 4: Output superstep for outputting results.
    this.outputstep();
    watcher.stop();
    LOG.info("{} MasterService output step cost: {}", this, TimeUtil.readableTime(watcher.getTime()));
}