Examples with MasterService com.baidu.hugegraph.computer.core.master.MasterService used on opensource projects

Example 1 with MasterService

use of com.baidu.hugegraph.computer.core.master.MasterService in project hugegraph-computer by hugegraph.

the class WorkerServiceTest method testServiceWith1Worker.

@Test
public void testServiceWith1Worker() throws InterruptedException {
    ExecutorService pool = Executors.newFixedThreadPool(2);
    CountDownLatch countDownLatch = new CountDownLatch(2);
    Throwable[] exceptions = new Throwable[2];
    pool.submit(() -> {
        Config config = UnitTestBase.updateWithRequiredOptions(RpcOptions.RPC_REMOTE_URL, "127.0.0.1:8090", ComputerOptions.JOB_ID, "local_002", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.TRANSPORT_SERVER_PORT, "8086", ComputerOptions.BSP_REGISTER_TIMEOUT, "100000", ComputerOptions.BSP_LOG_INTERVAL, "30000", ComputerOptions.BSP_MAX_SUPER_STEP, "2", ComputerOptions.WORKER_COMPUTATION_CLASS, MockComputation.class.getName(), ComputerOptions.ALGORITHM_RESULT_CLASS, DoubleValue.class.getName(), ComputerOptions.ALGORITHM_MESSAGE_CLASS, DoubleValue.class.getName(), ComputerOptions.OUTPUT_CLASS, LimitedLogOutput.class.getName());
        WorkerService workerService = new MockWorkerService();
        try {
            workerService.init(config);
            workerService.execute();
        } catch (Throwable e) {
            LOG.error("Failed to start worker", e);
            exceptions[0] = e;
        } finally {
            workerService.close();
            try {
                workerService.close();
            } catch (Throwable e) {
                Assert.fail(e.getMessage());
            }
            countDownLatch.countDown();
        }
    });
    pool.submit(() -> {
        Config config = UnitTestBase.updateWithRequiredOptions(RpcOptions.RPC_SERVER_HOST, "localhost", RpcOptions.RPC_SERVER_PORT, "8090", ComputerOptions.JOB_ID, "local_002", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.BSP_REGISTER_TIMEOUT, "100000", ComputerOptions.BSP_LOG_INTERVAL, "30000", ComputerOptions.BSP_MAX_SUPER_STEP, "2", ComputerOptions.MASTER_COMPUTATION_CLASS, MockMasterComputation.class.getName(), ComputerOptions.ALGORITHM_RESULT_CLASS, DoubleValue.class.getName(), ComputerOptions.ALGORITHM_MESSAGE_CLASS, DoubleValue.class.getName());
        MasterService masterService = new MasterService();
        try {
            masterService.init(config);
            masterService.execute();
        } catch (Throwable e) {
            LOG.error("Failed to start master", e);
            exceptions[1] = e;
        } finally {
            /*
                 * It must close the service first. The pool will be shutdown
                 * if count down is executed first, and the server thread in
                 * master service will not be closed.
                 */
            masterService.close();
            try {
                masterService.close();
            } catch (Throwable e) {
                Assert.fail(e.getMessage());
            }
            countDownLatch.countDown();
        }
    });
    countDownLatch.await();
    pool.shutdownNow();
    Assert.assertFalse(Arrays.asList(exceptions).toString(), existError(exceptions));
}

Example 2 with MasterService

use of com.baidu.hugegraph.computer.core.master.MasterService in project hugegraph-computer by hugegraph.

the class WorkerServiceTest method testServiceWith2Workers.

@Test
public void testServiceWith2Workers() throws InterruptedException {
    ExecutorService pool = Executors.newFixedThreadPool(3);
    CountDownLatch countDownLatch = new CountDownLatch(3);
    Throwable[] exceptions = new Throwable[3];
    pool.submit(() -> {
        Config config = UnitTestBase.updateWithRequiredOptions(RpcOptions.RPC_REMOTE_URL, "127.0.0.1:8090", ComputerOptions.JOB_ID, "local_003", ComputerOptions.JOB_WORKERS_COUNT, "2", ComputerOptions.JOB_PARTITIONS_COUNT, "2", ComputerOptions.TRANSPORT_SERVER_PORT, "8086", ComputerOptions.WORKER_DATA_DIRS, "[job_8086]", ComputerOptions.BSP_REGISTER_TIMEOUT, "30000", ComputerOptions.BSP_LOG_INTERVAL, "10000", ComputerOptions.BSP_MAX_SUPER_STEP, "2", ComputerOptions.WORKER_COMPUTATION_CLASS, MockComputation2.class.getName(), ComputerOptions.ALGORITHM_RESULT_CLASS, DoubleValue.class.getName(), ComputerOptions.ALGORITHM_MESSAGE_CLASS, DoubleValue.class.getName());
        WorkerService workerService = new MockWorkerService();
        try {
            workerService.init(config);
            workerService.execute();
        } catch (Throwable e) {
            LOG.error("Failed to start worker", e);
            exceptions[0] = e;
        } finally {
            workerService.close();
            countDownLatch.countDown();
        }
    });
    pool.submit(() -> {
        Config config = UnitTestBase.updateWithRequiredOptions(RpcOptions.RPC_REMOTE_URL, "127.0.0.1:8090", ComputerOptions.JOB_ID, "local_003", ComputerOptions.JOB_WORKERS_COUNT, "2", ComputerOptions.JOB_PARTITIONS_COUNT, "2", ComputerOptions.TRANSPORT_SERVER_PORT, "8087", ComputerOptions.WORKER_DATA_DIRS, "[job_8087]", ComputerOptions.BSP_REGISTER_TIMEOUT, "30000", ComputerOptions.BSP_LOG_INTERVAL, "10000", ComputerOptions.BSP_MAX_SUPER_STEP, "2", ComputerOptions.WORKER_COMPUTATION_CLASS, MockComputation2.class.getName(), ComputerOptions.ALGORITHM_RESULT_CLASS, DoubleValue.class.getName(), ComputerOptions.ALGORITHM_MESSAGE_CLASS, DoubleValue.class.getName());
        WorkerService workerService = new MockWorkerService();
        try {
            workerService.init(config);
            workerService.execute();
        } catch (Throwable e) {
            LOG.error("Failed to start worker", e);
            exceptions[1] = e;
        } finally {
            workerService.close();
            countDownLatch.countDown();
        }
    });
    pool.submit(() -> {
        Config config = UnitTestBase.updateWithRequiredOptions(RpcOptions.RPC_SERVER_HOST, "localhost", RpcOptions.RPC_SERVER_PORT, "8090", ComputerOptions.JOB_ID, "local_003", ComputerOptions.JOB_WORKERS_COUNT, "2", ComputerOptions.JOB_PARTITIONS_COUNT, "2", ComputerOptions.BSP_REGISTER_TIMEOUT, "30000", ComputerOptions.BSP_LOG_INTERVAL, "10000", ComputerOptions.BSP_MAX_SUPER_STEP, "2", ComputerOptions.MASTER_COMPUTATION_CLASS, MockMasterComputation2.class.getName(), ComputerOptions.ALGORITHM_RESULT_CLASS, DoubleValue.class.getName(), ComputerOptions.ALGORITHM_MESSAGE_CLASS, DoubleValue.class.getName());
        MasterService masterService = new MasterService();
        try {
            masterService.init(config);
            masterService.execute();
        } catch (Throwable e) {
            LOG.error("Failed to start master", e);
            exceptions[2] = e;
        } finally {
            masterService.close();
            countDownLatch.countDown();
        }
    });
    countDownLatch.await();
    pool.shutdownNow();
    Assert.assertFalse(Arrays.asList(exceptions).toString(), existError(exceptions));
}

Example 3 with MasterService

use of com.baidu.hugegraph.computer.core.master.MasterService in project hugegraph-computer by hugegraph.

the class SenderIntegrateTest method initMaster.

private MasterService initMaster(String[] args) {
    Config config = ComputerContextUtil.initContext(ComputerContextUtil.convertToMap(args));
    MasterService service = new MasterService();
    service.init(config);
    return service;
}

Example 4 with MasterService

use of com.baidu.hugegraph.computer.core.master.MasterService in project hugegraph-computer by hugegraph.

the class SenderIntegrateTest method testOneWorkerWithBusyClient.

@Test
public void testOneWorkerWithBusyClient() throws InterruptedException {
    Thread masterThread = new Thread(() -> {
        String[] args = OptionsBuilder.newInstance().withJobId("local_002").withAlgorithm(PageRankParams.class).withResultName("rank").withResultClass(DoubleValue.class).withMessageClass(DoubleValue.class).withMaxSuperStep(3).withComputationClass(COMPUTATION).withWorkerCount(1).withWriteBufferHighMark(10).withWriteBufferLowMark(5).withRpcServerHost("127.0.0.1").withRpcServerPort(8090).build();
        try (MasterService service = initMaster(args)) {
            service.execute();
        } catch (Exception e) {
            LOG.error("Failed to execute master service", e);
            Assert.fail(e.getMessage());
        }
    });
    Thread workerThread = new Thread(() -> {
        String[] args = OptionsBuilder.newInstance().withJobId("local_002").withAlgorithm(PageRankParams.class).withResultName("rank").withResultClass(DoubleValue.class).withMessageClass(DoubleValue.class).withMaxSuperStep(3).withComputationClass(COMPUTATION).withWorkerCount(1).withTransoprtServerPort(8091).withWriteBufferHighMark(20).withWriteBufferLowMark(10).withRpcServerRemote("127.0.0.1:8090").build();
        try (WorkerService service = initWorker(args)) {
            // Let send rate slowly
            this.slowSendFunc(service);
            service.execute();
        } catch (Exception e) {
            LOG.error("Failed to execute worker service", e);
            Assert.fail(e.getMessage());
        }
    });
    masterThread.start();
    workerThread.start();
    workerThread.join();
    masterThread.join();
}

Example 5 with MasterService

use of com.baidu.hugegraph.computer.core.master.MasterService in project hugegraph-computer by hugegraph.

the class SenderIntegrateTest method testOneWorker.

@Test
public void testOneWorker() throws InterruptedException {
    Thread masterThread = new Thread(() -> {
        String[] args = OptionsBuilder.newInstance().withJobId("local_002").withAlgorithm(PageRankParams.class).withResultName("rank").withResultClass(DoubleValue.class).withMessageClass(DoubleValue.class).withMaxSuperStep(3).withComputationClass(COMPUTATION).withWorkerCount(1).withBufferThreshold(50).withBufferCapacity(60).withRpcServerHost("127.0.0.1").withRpcServerPort(8090).build();
        try (MasterService service = initMaster(args)) {
            service.execute();
        } catch (Exception e) {
            Assert.fail(e.getMessage());
        }
    });
    Thread workerThread = new Thread(() -> {
        String[] args = OptionsBuilder.newInstance().withJobId("local_002").withAlgorithm(PageRankParams.class).withResultName("rank").withResultClass(DoubleValue.class).withMessageClass(DoubleValue.class).withMaxSuperStep(3).withComputationClass(COMPUTATION).withWorkerCount(1).withBufferThreshold(50).withBufferCapacity(60).withTransoprtServerPort(8091).withRpcServerRemote("127.0.0.1:8090").build();
        try (WorkerService service = initWorker(args)) {
            service.execute();
        } catch (Exception e) {
            Assert.fail(e.getMessage());
        }
    });
    masterThread.start();
    workerThread.start();
    workerThread.join();
    masterThread.join();
}