use of com.baidu.hugegraph.computer.core.worker.WorkerService in project hugegraph-computer by hugegraph.
the class SenderIntegrateTest method initWorker.
private WorkerService initWorker(String[] args) {
Config config = ComputerContextUtil.initContext(ComputerContextUtil.convertToMap(args));
WorkerService service = new WorkerService();
service.init(config);
return service;
}
use of com.baidu.hugegraph.computer.core.worker.WorkerService in project hugegraph-computer by hugegraph.
the class SenderIntegrateTest method testOneWorkerWithBusyClient.
@Test
public void testOneWorkerWithBusyClient() throws InterruptedException {
Thread masterThread = new Thread(() -> {
String[] args = OptionsBuilder.newInstance().withJobId("local_002").withAlgorithm(PageRankParams.class).withResultName("rank").withResultClass(DoubleValue.class).withMessageClass(DoubleValue.class).withMaxSuperStep(3).withComputationClass(COMPUTATION).withWorkerCount(1).withWriteBufferHighMark(10).withWriteBufferLowMark(5).withRpcServerHost("127.0.0.1").withRpcServerPort(8090).build();
try (MasterService service = initMaster(args)) {
service.execute();
} catch (Exception e) {
LOG.error("Failed to execute master service", e);
Assert.fail(e.getMessage());
}
});
Thread workerThread = new Thread(() -> {
String[] args = OptionsBuilder.newInstance().withJobId("local_002").withAlgorithm(PageRankParams.class).withResultName("rank").withResultClass(DoubleValue.class).withMessageClass(DoubleValue.class).withMaxSuperStep(3).withComputationClass(COMPUTATION).withWorkerCount(1).withTransoprtServerPort(8091).withWriteBufferHighMark(20).withWriteBufferLowMark(10).withRpcServerRemote("127.0.0.1:8090").build();
try (WorkerService service = initWorker(args)) {
// Let send rate slowly
this.slowSendFunc(service);
service.execute();
} catch (Exception e) {
LOG.error("Failed to execute worker service", e);
Assert.fail(e.getMessage());
}
});
masterThread.start();
workerThread.start();
workerThread.join();
masterThread.join();
}
use of com.baidu.hugegraph.computer.core.worker.WorkerService in project hugegraph-computer by hugegraph.
the class SenderIntegrateTest method testOneWorker.
@Test
public void testOneWorker() throws InterruptedException {
Thread masterThread = new Thread(() -> {
String[] args = OptionsBuilder.newInstance().withJobId("local_002").withAlgorithm(PageRankParams.class).withResultName("rank").withResultClass(DoubleValue.class).withMessageClass(DoubleValue.class).withMaxSuperStep(3).withComputationClass(COMPUTATION).withWorkerCount(1).withBufferThreshold(50).withBufferCapacity(60).withRpcServerHost("127.0.0.1").withRpcServerPort(8090).build();
try (MasterService service = initMaster(args)) {
service.execute();
} catch (Exception e) {
Assert.fail(e.getMessage());
}
});
Thread workerThread = new Thread(() -> {
String[] args = OptionsBuilder.newInstance().withJobId("local_002").withAlgorithm(PageRankParams.class).withResultName("rank").withResultClass(DoubleValue.class).withMessageClass(DoubleValue.class).withMaxSuperStep(3).withComputationClass(COMPUTATION).withWorkerCount(1).withBufferThreshold(50).withBufferCapacity(60).withTransoprtServerPort(8091).withRpcServerRemote("127.0.0.1:8090").build();
try (WorkerService service = initWorker(args)) {
service.execute();
} catch (Exception e) {
Assert.fail(e.getMessage());
}
});
masterThread.start();
workerThread.start();
workerThread.join();
masterThread.join();
}
use of com.baidu.hugegraph.computer.core.worker.WorkerService in project hugegraph-computer by hugegraph.
the class HugeGraphComputer method executeWorkerService.
private static void executeWorkerService(ComputerContext context) {
try (WorkerService workerService = new WorkerService()) {
workerService.init(context.config());
workerService.execute();
}
}
use of com.baidu.hugegraph.computer.core.worker.WorkerService in project hugegraph-computer by hugegraph.
the class AlgorithmTestBase method runAlgorithm.
public static void runAlgorithm(String algorithmParams, String... options) throws InterruptedException {
final Logger log = Log.logger(AlgorithmTestBase.class);
ExecutorService pool = Executors.newFixedThreadPool(2);
CountDownLatch countDownLatch = new CountDownLatch(2);
Throwable[] exceptions = new Throwable[2];
pool.submit(() -> {
WorkerService workerService = null;
try {
Map<String, String> params = new HashMap<>();
params.put(RpcOptions.RPC_REMOTE_URL.name(), "127.0.0.1:8090");
params.put(ComputerOptions.JOB_ID.name(), "algo_test_job1");
params.put(ComputerOptions.JOB_WORKERS_COUNT.name(), "1");
params.put(ComputerOptions.TRANSPORT_SERVER_PORT.name(), "8086");
params.put(ComputerOptions.BSP_REGISTER_TIMEOUT.name(), "100000");
params.put(ComputerOptions.BSP_LOG_INTERVAL.name(), "30000");
params.put(ComputerOptions.BSP_MAX_SUPER_STEP.name(), "10");
params.put(ComputerOptions.ALGORITHM_PARAMS_CLASS.name(), algorithmParams);
if (options != null) {
for (int i = 0; i < options.length; i += 2) {
params.put(options[i], options[i + 1]);
}
}
Config config = ComputerContextUtil.initContext(params);
workerService = new MockWorkerService();
workerService.init(config);
workerService.execute();
} catch (Throwable e) {
LOG.error("Failed to start worker", e);
exceptions[0] = e;
// If worker failed, the master also should quit
while (countDownLatch.getCount() > 0) {
countDownLatch.countDown();
}
} finally {
if (workerService != null) {
workerService.close();
}
countDownLatch.countDown();
}
});
pool.submit(() -> {
MasterService masterService = null;
try {
Map<String, String> params = new HashMap<>();
params.put(RpcOptions.RPC_SERVER_HOST.name(), "localhost");
params.put(RpcOptions.RPC_SERVER_PORT.name(), "8090");
params.put(ComputerOptions.JOB_ID.name(), "algo_test_job1");
params.put(ComputerOptions.JOB_WORKERS_COUNT.name(), "1");
params.put(ComputerOptions.BSP_REGISTER_TIMEOUT.name(), "100000");
params.put(ComputerOptions.BSP_LOG_INTERVAL.name(), "30000");
params.put(ComputerOptions.BSP_MAX_SUPER_STEP.name(), "10");
params.put(ComputerOptions.ALGORITHM_PARAMS_CLASS.name(), algorithmParams);
if (options != null) {
for (int i = 0; i < options.length; i += 2) {
params.put(options[i], options[i + 1]);
}
}
Config config = ComputerContextUtil.initContext(params);
masterService = new MasterService();
masterService.init(config);
masterService.execute();
} catch (Throwable e) {
LOG.error("Failed to start master", e);
exceptions[1] = e;
// If master failed, the worker also should quit
while (countDownLatch.getCount() > 0) {
countDownLatch.countDown();
}
} finally {
/*
* It must close the service first. The pool will be shutdown
* if count down is executed first, and the server thread in
* master service will not be closed.
*/
if (masterService != null) {
masterService.close();
}
countDownLatch.countDown();
}
});
countDownLatch.await();
pool.shutdownNow();
Assert.assertFalse(Arrays.asList(exceptions).toString(), existError(exceptions));
}
Aggregations