Examples with Config com.baidu.hugegraph.computer.core.config.Config used on opensource projects

Example 21 with Config

use of com.baidu.hugegraph.computer.core.config.Config in project hugegraph-computer by hugegraph.

the class AlgorithmTestBase method runAlgorithm.

public static void runAlgorithm(String algorithmParams, String... options) throws InterruptedException {
    final Logger log = Log.logger(AlgorithmTestBase.class);
    ExecutorService pool = Executors.newFixedThreadPool(2);
    CountDownLatch countDownLatch = new CountDownLatch(2);
    Throwable[] exceptions = new Throwable[2];
    pool.submit(() -> {
        WorkerService workerService = null;
        try {
            Map<String, String> params = new HashMap<>();
            params.put(RpcOptions.RPC_REMOTE_URL.name(), "127.0.0.1:8090");
            params.put(ComputerOptions.JOB_ID.name(), "algo_test_job1");
            params.put(ComputerOptions.JOB_WORKERS_COUNT.name(), "1");
            params.put(ComputerOptions.TRANSPORT_SERVER_PORT.name(), "8086");
            params.put(ComputerOptions.BSP_REGISTER_TIMEOUT.name(), "100000");
            params.put(ComputerOptions.BSP_LOG_INTERVAL.name(), "30000");
            params.put(ComputerOptions.BSP_MAX_SUPER_STEP.name(), "10");
            params.put(ComputerOptions.ALGORITHM_PARAMS_CLASS.name(), algorithmParams);
            if (options != null) {
                for (int i = 0; i < options.length; i += 2) {
                    params.put(options[i], options[i + 1]);
                }
            }
            Config config = ComputerContextUtil.initContext(params);
            workerService = new MockWorkerService();
            workerService.init(config);
            workerService.execute();
        } catch (Throwable e) {
            LOG.error("Failed to start worker", e);
            exceptions[0] = e;
            // If worker failed, the master also should quit
            while (countDownLatch.getCount() > 0) {
                countDownLatch.countDown();
            }
        } finally {
            if (workerService != null) {
                workerService.close();
            }
            countDownLatch.countDown();
        }
    });
    pool.submit(() -> {
        MasterService masterService = null;
        try {
            Map<String, String> params = new HashMap<>();
            params.put(RpcOptions.RPC_SERVER_HOST.name(), "localhost");
            params.put(RpcOptions.RPC_SERVER_PORT.name(), "8090");
            params.put(ComputerOptions.JOB_ID.name(), "algo_test_job1");
            params.put(ComputerOptions.JOB_WORKERS_COUNT.name(), "1");
            params.put(ComputerOptions.BSP_REGISTER_TIMEOUT.name(), "100000");
            params.put(ComputerOptions.BSP_LOG_INTERVAL.name(), "30000");
            params.put(ComputerOptions.BSP_MAX_SUPER_STEP.name(), "10");
            params.put(ComputerOptions.ALGORITHM_PARAMS_CLASS.name(), algorithmParams);
            if (options != null) {
                for (int i = 0; i < options.length; i += 2) {
                    params.put(options[i], options[i + 1]);
                }
            }
            Config config = ComputerContextUtil.initContext(params);
            masterService = new MasterService();
            masterService.init(config);
            masterService.execute();
        } catch (Throwable e) {
            LOG.error("Failed to start master", e);
            exceptions[1] = e;
            // If master failed, the worker also should quit
            while (countDownLatch.getCount() > 0) {
                countDownLatch.countDown();
            }
        } finally {
            /*
                 * It must close the service first. The pool will be shutdown
                 * if count down is executed first, and the server thread in
                 * master service will not be closed.
                 */
            if (masterService != null) {
                masterService.close();
            }
            countDownLatch.countDown();
        }
    });
    countDownLatch.await();
    pool.shutdownNow();
    Assert.assertFalse(Arrays.asList(exceptions).toString(), existError(exceptions));
}

Example 22 with Config

use of com.baidu.hugegraph.computer.core.config.Config in project hugegraph-computer by hugegraph.

the class PointerCombinerTest method testVertexPropertiesCombiner.

@Test
public void testVertexPropertiesCombiner() throws IOException {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.WORKER_COMBINER_CLASS, DoubleValueSumCombiner.class.getName(), ComputerOptions.WORKER_VERTEX_PROPERTIES_COMBINER_CLASS, MergeOldPropertiesCombiner.class.getName());
    Combiner<Properties> valueCombiner = config.createObject(ComputerOptions.WORKER_VERTEX_PROPERTIES_COMBINER_CLASS);
    GraphFactory graphFactory = graphFactory();
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(graphFactory::createProperties, valueCombiner);
    try (BytesOutput bytesOutput1 = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
        BytesOutput bytesOutput2 = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE)) {
        Properties value1 = graphFactory.createProperties();
        value1.put("p1", new LongValue(1L));
        Properties value2 = graphFactory.createProperties();
        value2.put("p2", new LongValue(2L));
        value1.write(bytesOutput1);
        value2.write(bytesOutput2);
        Pointer pointer1 = new InlinePointer(bytesOutput1.buffer(), bytesOutput1.position());
        Pointer pointer2 = new InlinePointer(bytesOutput2.buffer(), bytesOutput2.position());
        Pointer pointer = combiner.combine(pointer1, pointer2);
        BytesInput input = IOFactory.createBytesInput(pointer.bytes());
        Properties combinedValue = graphFactory.createProperties();
        combinedValue.read(input);
        Map<String, Value> map = combinedValue.get();
        Assert.assertEquals(2, map.size());
        Assert.assertEquals(new LongValue(1L), map.get("p1"));
        Assert.assertEquals(new LongValue(2L), map.get("p2"));
    }
}

Example 23 with Config

use of com.baidu.hugegraph.computer.core.config.Config in project hugegraph-computer by hugegraph.

the class PointerCombinerTest method testCombineEdgePropertiesFail.

@Test
public void testCombineEdgePropertiesFail() throws IOException {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.WORKER_COMBINER_CLASS, DoubleValueSumCombiner.class.getName(), ComputerOptions.WORKER_EDGE_PROPERTIES_COMBINER_CLASS, MergeOldPropertiesCombiner.class.getName());
    Combiner<Properties> valueCombiner = config.createObject(ComputerOptions.WORKER_EDGE_PROPERTIES_COMBINER_CLASS);
    GraphFactory graphFactory = graphFactory();
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(graphFactory::createProperties, valueCombiner);
    try (BytesOutput bytesOutput1 = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
        BytesOutput bytesOutput2 = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE)) {
        Properties value1 = graphFactory.createProperties();
        value1.put("p1", new LongValue(1L));
        Properties value2 = graphFactory.createProperties();
        value2.put("p2", new LongValue(2L));
        // Only write count.
        bytesOutput1.writeInt(1);
        value2.write(bytesOutput2);
        Pointer pointer1 = new InlinePointer(bytesOutput1.buffer(), bytesOutput1.position());
        Pointer pointer2 = new InlinePointer(bytesOutput2.buffer(), bytesOutput2.position());
        Assert.assertThrows(ComputerException.class, () -> {
            combiner.combine(pointer1, pointer2);
        }, e -> {
            Assert.assertContains("Failed to combine pointer", e.getMessage());
        });
    }
}

Example 24 with Config

use of com.baidu.hugegraph.computer.core.config.Config in project hugegraph-computer by hugegraph.

the class EtcdBspTest method setup.

@Before
public void setup() {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.BSP_LOG_INTERVAL, "30000", ComputerOptions.BSP_MAX_SUPER_STEP, "2");
    this.bsp4Master = new Bsp4Master(config);
    this.bsp4Master.clean();
    this.masterInfo = new ContainerInfo(-1, "localhost", 8001, 8002);
    this.workerInfo = new ContainerInfo(0, "localhost", 8003, 8004);
    this.bsp4Worker = new Bsp4Worker(config, this.workerInfo);
    this.maxSuperStep = config.get(ComputerOptions.BSP_MAX_SUPER_STEP);
}

Example 25 with Config

use of com.baidu.hugegraph.computer.core.config.Config in project hugegraph-computer by hugegraph.

the class ComputeMessageRecvPartitionTest method testNotCombineMessageRecvPartition.

@Test
public void testNotCombineMessageRecvPartition() throws IOException {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.JOB_ID, "local_001", ComputerOptions.JOB_WORKERS_COUNT, "1", ComputerOptions.JOB_PARTITIONS_COUNT, "1", ComputerOptions.WORKER_COMBINER_CLASS, Null.class.getName(), ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10", ComputerOptions.ALGORITHM_MESSAGE_CLASS, IdList.class.getName(), ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    FileUtils.deleteQuietly(new File("data_dir1"));
    FileUtils.deleteQuietly(new File("data_dir2"));
    FileManager fileManager = new FileManager();
    fileManager.init(config);
    SortManager sortManager = new RecvSortManager(context());
    sortManager.init(config);
    SuperstepFileGenerator fileGenerator = new SuperstepFileGenerator(fileManager, 0);
    ComputeMessageRecvPartition partition = new ComputeMessageRecvPartition(context(), fileGenerator, sortManager);
    Assert.assertEquals("msg", partition.type());
    addTwentyDuplicateIdValueListMessageBuffer(partition::addBuffer);
    checkIdValueListMessages(partition.iterator());
    fileManager.close(config);
    sortManager.close(config);
}