Examples with JnaGraphStore com.alibaba.graphscope.groot.store.jna.JnaGraphStore used on opensource projects

Example 1 with JnaGraphStore

use of com.alibaba.graphscope.groot.store.jna.JnaGraphStore in project GraphScope by alibaba.

the class ExecutorService method start.

public void start() {
    logger.info("Start to launch executor service");
    int nodeCount = CommonConfig.STORE_NODE_COUNT.get(this.configs);
    int workerPerProcess = ExecutorConfig.EXECUTOR_WORKER_PER_PROCESS.get(this.configs);
    Configs executorConfig = Configs.newBuilder().put("node.idx", String.valueOf(CommonConfig.NODE_IDX.get(configs))).put("graph.name", CommonConfig.GRAPH_NAME.get(configs)).put("partition.count", String.valueOf(CommonConfig.PARTITION_COUNT.get(configs))).put("graph.port", String.valueOf(StoreConfig.EXECUTOR_GRAPH_PORT.get(configs))).put("query.port", String.valueOf(StoreConfig.EXECUTOR_QUERY_PORT.get(configs))).put("engine.port", String.valueOf(StoreConfig.EXECUTOR_ENGINE_PORT.get(configs))).put("worker.per.process", String.valueOf(workerPerProcess)).put("worker.num", String.valueOf(nodeCount)).build();
    byte[] configBytes = executorConfig.toProto().toByteArray();
    logger.info("Start to open executor server");
    Pointer pointer = ExecutorLibrary.INSTANCE.openExecutorServer(configBytes, configBytes.length);
    // Add graph store with partition id to executor
    Map<Integer, GraphPartition> idToPartition = this.storeService.getIdToPartition();
    logger.info("Get partition count " + idToPartition.size() + " for current store node");
    for (Map.Entry<Integer, GraphPartition> partition : idToPartition.entrySet()) {
        int partitionId = partition.getKey();
        GraphPartition graphStore = partition.getValue();
        if (graphStore instanceof JnaGraphStore) {
            JnaGraphStore jnaGraphStore = (JnaGraphStore) graphStore;
            ExecutorLibrary.INSTANCE.addGraphPartition(pointer, partitionId, jnaGraphStore.getPointer());
            logger.info("Add partition " + partitionId);
        }
    }
    // Assign partition id to each worker id
    for (int i = 0; i < nodeCount; i++) {
        List<Integer> partitionIdList = metaService.getPartitionsByStoreId(i);
        logger.info("Get partition id list " + partitionIdList + " for store index " + i);
        partitionIdList.sort(Integer::compareTo);
        for (int k = 0; k < partitionIdList.size(); k++) {
            int partitionId = partitionIdList.get(k);
            int innnerIndex = k % workerPerProcess;
            int workerId = (i * workerPerProcess) + innnerIndex;
            logger.info("Add partition->worker mapping " + partitionId + "->" + workerId);
            ExecutorLibrary.INSTANCE.addPartitionWorkerMapping(pointer, partitionId, workerId);
        }
    }
    this.executorManager.initialExecutor(pointer, nodeCount);
    this.executorManager.startEngineServer();
    this.executorManager.startRpcServer();
    this.executorManager.getDiscoveryManager().getEngineServerDiscovery().addListener(new EngineServerListener(this.executorManager));
    while (!this.executorManager.checkEngineServersConnect()) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
        logger.info("Wait all the engine server to build connection with each other");
    }
    logger.info("All engine server build connection with each success");
}

Example 2 with JnaGraphStore

use of com.alibaba.graphscope.groot.store.jna.JnaGraphStore in project GraphScope by alibaba.

the class FfiTest method testFfi.

@Test
public void testFfi() throws Exception {
    GraphProvider provider = new MaxTestGraphProvider();
    LoadGraphWith.GraphData modern = LoadGraphWith.GraphData.MODERN;
    Map<String, Object> conf = new HashMap<>();
    conf.put(CommonConfig.STORE_NODE_COUNT.getKey(), "1");
    conf.put(CommonConfig.PARTITION_COUNT.getKey(), "1");
    Configuration graphConf = provider.newGraphConfiguration("ffi-test", FfiTest.class, "testFfi", conf, modern);
    provider.clear(graphConf);
    Graph graph = provider.openTestGraph(graphConf);
    LoadGraphWith loadGraphWith = new LoadGraphWith() {

        @Override
        public Class<? extends Annotation> annotationType() {
            return null;
        }

        @Override
        public GraphData value() {
            return modern;
        }
    };
    provider.loadGraphData(graph, loadGraphWith, FfiTest.class, "testFfi");
    MaxNode maxNode = ((MaxTestGraph) graph).getMaxNode();
    List<NodeBase> storeNodes = maxNode.getStores();
    assertEquals(storeNodes.size(), 1);
    Store store = (Store) storeNodes.get(0);
    StoreService storeService = store.getStoreService();
    JnaGraphStore jnaGraphStore = (JnaGraphStore) storeService.getIdToPartition().get(0);
    Pointer wrapperPartitionGraph = GraphLibrary.INSTANCE.createWrapperPartitionGraph(jnaGraphStore.getPointer());
    GnnLibrary.INSTANCE.setPartitionGraph(wrapperPartitionGraph);
    GnnLibrary.TestResult testResult = new GnnLibrary.TestResult(GnnLibrary.INSTANCE.runLocalTests());
    logger.info(testResult.getInfo());
    assertTrue(testResult.getFlag());
    provider.clear(graph, graphConf);
    maxNode.close();
}

Example 3 with JnaGraphStore

use of com.alibaba.graphscope.groot.store.jna.JnaGraphStore in project GraphScope by alibaba.

the class BackupAgentTest method testBackupAgent.

@Test
void testBackupAgent() throws IOException {
    Configs configs = Configs.newBuilder().put(CommonConfig.NODE_IDX.getKey(), "0").put(BackupConfig.BACKUP_ENABLE.getKey(), "true").put(BackupConfig.STORE_BACKUP_THREAD_COUNT.getKey(), "2").build();
    StoreService mockStoreService = mock(StoreService.class);
    JnaGraphStore mockJnaStore0 = mock(JnaGraphStore.class);
    JnaGraphStore mockJnaStore1 = mock(JnaGraphStore.class);
    JnaGraphBackupEngine mockJnaBackupEngine0 = mock(JnaGraphBackupEngine.class);
    JnaGraphBackupEngine mockJnaBackupEngine1 = mock(JnaGraphBackupEngine.class);
    Map<Integer, GraphPartition> idToPartition = new HashMap<>();
    idToPartition.put(0, mockJnaStore0);
    idToPartition.put(1, mockJnaStore1);
    when(mockStoreService.getIdToPartition()).thenReturn(idToPartition);
    when(mockJnaStore0.openBackupEngine()).thenReturn(mockJnaBackupEngine0);
    when(mockJnaStore1.openBackupEngine()).thenReturn(mockJnaBackupEngine1);
    BackupAgent backupAgent = new BackupAgent(configs, mockStoreService);
    backupAgent.start();
    StoreBackupId storeBackupId = new StoreBackupId(5);
    storeBackupId.addPartitionBackupId(0, 7);
    storeBackupId.addPartitionBackupId(1, 6);
    Map<Integer, List<Integer>> readyPartitionBackupIds = new HashMap<>();
    readyPartitionBackupIds.put(0, Arrays.asList(2, 4, 6));
    readyPartitionBackupIds.put(1, Arrays.asList(3, 5, 7));
    when(mockJnaBackupEngine0.createNewPartitionBackup()).thenReturn(7);
    when(mockJnaBackupEngine1.createNewPartitionBackup()).thenReturn(6);
    CompletionCallback<StoreBackupId> createCallback = mock(CompletionCallback.class);
    backupAgent.createNewStoreBackup(5, createCallback);
    verify(createCallback, timeout(5000L)).onCompleted(storeBackupId);
    CompletionCallback<Void> verifyCallback = mock(CompletionCallback.class);
    backupAgent.verifyStoreBackup(storeBackupId, verifyCallback);
    verify(mockJnaBackupEngine0, timeout(5000L)).verifyPartitionBackup(7);
    verify(mockJnaBackupEngine1, timeout(5000L)).verifyPartitionBackup(6);
    verify(verifyCallback, timeout(5000L)).onCompleted(null);
    CompletionCallback<Void> clearCallback = mock(CompletionCallback.class);
    backupAgent.clearUnavailableStoreBackups(readyPartitionBackupIds, clearCallback);
    verify(mockJnaBackupEngine0, timeout(5000L)).partitionBackupGc(Arrays.asList(2, 4, 6));
    verify(mockJnaBackupEngine1, timeout(5000L)).partitionBackupGc(Arrays.asList(3, 5, 7));
    verify(clearCallback, timeout(5000L)).onCompleted(null);
    CompletionCallback<Void> restoreCallback = mock(CompletionCallback.class);
    backupAgent.restoreFromStoreBackup(storeBackupId, "restore_root", restoreCallback);
    verify(mockJnaBackupEngine0, timeout(5000L)).restoreFromPartitionBackup(7, Paths.get("restore_root", "0").toString());
    verify(mockJnaBackupEngine1, timeout(5000L)).restoreFromPartitionBackup(6, Paths.get("restore_root", "1").toString());
    verify(restoreCallback, timeout(5000L)).onCompleted(null);
    backupAgent.stop();
}