Examples with GraphPartition com.alibaba.graphscope.groot.store.GraphPartition used on opensource projects

Example 1 with GraphPartition

use of com.alibaba.graphscope.groot.store.GraphPartition in project GraphScope by alibaba.

the class GaiaService method start.

@Override
public void start() {
    this.engine.init();
    for (GraphPartition partition : this.storeService.getIdToPartition().values()) {
        this.engine.addPartition(partition);
    }
    int partitionCount = CommonConfig.PARTITION_COUNT.get(this.configs);
    for (int i = 0; i < partitionCount; i++) {
        this.engine.updatePartitionRouting(i, metaService.getStoreIdByPartition(i));
    }
    this.engine.start();
}

Example 2 with GraphPartition

use of com.alibaba.graphscope.groot.store.GraphPartition in project GraphScope by alibaba.

the class ExecutorService method start.

public void start() {
    logger.info("Start to launch executor service");
    int nodeCount = CommonConfig.STORE_NODE_COUNT.get(this.configs);
    int workerPerProcess = ExecutorConfig.EXECUTOR_WORKER_PER_PROCESS.get(this.configs);
    Configs executorConfig = Configs.newBuilder().put("node.idx", String.valueOf(CommonConfig.NODE_IDX.get(configs))).put("graph.name", CommonConfig.GRAPH_NAME.get(configs)).put("partition.count", String.valueOf(CommonConfig.PARTITION_COUNT.get(configs))).put("graph.port", String.valueOf(StoreConfig.EXECUTOR_GRAPH_PORT.get(configs))).put("query.port", String.valueOf(StoreConfig.EXECUTOR_QUERY_PORT.get(configs))).put("engine.port", String.valueOf(StoreConfig.EXECUTOR_ENGINE_PORT.get(configs))).put("worker.per.process", String.valueOf(workerPerProcess)).put("worker.num", String.valueOf(nodeCount)).build();
    byte[] configBytes = executorConfig.toProto().toByteArray();
    logger.info("Start to open executor server");
    Pointer pointer = ExecutorLibrary.INSTANCE.openExecutorServer(configBytes, configBytes.length);
    // Add graph store with partition id to executor
    Map<Integer, GraphPartition> idToPartition = this.storeService.getIdToPartition();
    logger.info("Get partition count " + idToPartition.size() + " for current store node");
    for (Map.Entry<Integer, GraphPartition> partition : idToPartition.entrySet()) {
        int partitionId = partition.getKey();
        GraphPartition graphStore = partition.getValue();
        if (graphStore instanceof JnaGraphStore) {
            JnaGraphStore jnaGraphStore = (JnaGraphStore) graphStore;
            ExecutorLibrary.INSTANCE.addGraphPartition(pointer, partitionId, jnaGraphStore.getPointer());
            logger.info("Add partition " + partitionId);
        }
    }
    // Assign partition id to each worker id
    for (int i = 0; i < nodeCount; i++) {
        List<Integer> partitionIdList = metaService.getPartitionsByStoreId(i);
        logger.info("Get partition id list " + partitionIdList + " for store index " + i);
        partitionIdList.sort(Integer::compareTo);
        for (int k = 0; k < partitionIdList.size(); k++) {
            int partitionId = partitionIdList.get(k);
            int innnerIndex = k % workerPerProcess;
            int workerId = (i * workerPerProcess) + innnerIndex;
            logger.info("Add partition->worker mapping " + partitionId + "->" + workerId);
            ExecutorLibrary.INSTANCE.addPartitionWorkerMapping(pointer, partitionId, workerId);
        }
    }
    this.executorManager.initialExecutor(pointer, nodeCount);
    this.executorManager.startEngineServer();
    this.executorManager.startRpcServer();
    this.executorManager.getDiscoveryManager().getEngineServerDiscovery().addListener(new EngineServerListener(this.executorManager));
    while (!this.executorManager.checkEngineServersConnect()) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
        logger.info("Wait all the engine server to build connection with each other");
    }
    logger.info("All engine server build connection with each success");
}

Example 3 with GraphPartition

use of com.alibaba.graphscope.groot.store.GraphPartition in project GraphScope by alibaba.

the class GaiaService method start.

@Override
public void start() {
    this.engine.init();
    for (GraphPartition partition : this.storeService.getIdToPartition().values()) {
        this.engine.addPartition(partition);
    }
    int partitionCount = CommonConfig.PARTITION_COUNT.get(this.configs);
    for (int i = 0; i < partitionCount; i++) {
        this.engine.updatePartitionRouting(i, metaService.getStoreIdByPartition(i));
    }
    this.engine.start();
}

Example 4 with GraphPartition

use of com.alibaba.graphscope.groot.store.GraphPartition in project GraphScope by alibaba.

the class StoreServiceTest method testStoreService.

@Test
void testStoreService() throws IOException, InterruptedException, ExecutionException {
    Configs configs = Configs.newBuilder().put(CommonConfig.NODE_IDX.getKey(), "0").build();
    MetaService mockMetaService = mock(MetaService.class);
    when(mockMetaService.getPartitionsByStoreId(0)).thenReturn(Arrays.asList(0));
    StoreService spyStoreService = spy(new StoreService(configs, mockMetaService, new MetricsCollector(configs)));
    GraphPartition mockGraphPartition = mock(GraphPartition.class);
    when(mockGraphPartition.recover()).thenReturn(10L);
    doReturn(mockGraphPartition).when(spyStoreService).makeGraphPartition(any(), eq(0));
    spyStoreService.start();
    assertEquals(spyStoreService.recover(), 10L);
    StoreDataBatch storeDataBatch = StoreDataBatch.newBuilder().snapshotId(20L).addOperation(0, OperationBlob.MARKER_OPERATION_BLOB).build();
    spyStoreService.batchWrite(storeDataBatch);
    verify(mockGraphPartition, timeout(100L)).writeBatch(20L, OperationBatch.newBuilder().addOperationBlob(OperationBlob.MARKER_OPERATION_BLOB).build());
    spyStoreService.stop();
    verify(mockGraphPartition).close();
}

Example 5 with GraphPartition

use of com.alibaba.graphscope.groot.store.GraphPartition in project GraphScope by alibaba.

the class BackupAgentTest method testBackupAgent.

@Test
void testBackupAgent() throws IOException {
    Configs configs = Configs.newBuilder().put(CommonConfig.NODE_IDX.getKey(), "0").put(BackupConfig.BACKUP_ENABLE.getKey(), "true").put(BackupConfig.STORE_BACKUP_THREAD_COUNT.getKey(), "2").build();
    StoreService mockStoreService = mock(StoreService.class);
    JnaGraphStore mockJnaStore0 = mock(JnaGraphStore.class);
    JnaGraphStore mockJnaStore1 = mock(JnaGraphStore.class);
    JnaGraphBackupEngine mockJnaBackupEngine0 = mock(JnaGraphBackupEngine.class);
    JnaGraphBackupEngine mockJnaBackupEngine1 = mock(JnaGraphBackupEngine.class);
    Map<Integer, GraphPartition> idToPartition = new HashMap<>();
    idToPartition.put(0, mockJnaStore0);
    idToPartition.put(1, mockJnaStore1);
    when(mockStoreService.getIdToPartition()).thenReturn(idToPartition);
    when(mockJnaStore0.openBackupEngine()).thenReturn(mockJnaBackupEngine0);
    when(mockJnaStore1.openBackupEngine()).thenReturn(mockJnaBackupEngine1);
    BackupAgent backupAgent = new BackupAgent(configs, mockStoreService);
    backupAgent.start();
    StoreBackupId storeBackupId = new StoreBackupId(5);
    storeBackupId.addPartitionBackupId(0, 7);
    storeBackupId.addPartitionBackupId(1, 6);
    Map<Integer, List<Integer>> readyPartitionBackupIds = new HashMap<>();
    readyPartitionBackupIds.put(0, Arrays.asList(2, 4, 6));
    readyPartitionBackupIds.put(1, Arrays.asList(3, 5, 7));
    when(mockJnaBackupEngine0.createNewPartitionBackup()).thenReturn(7);
    when(mockJnaBackupEngine1.createNewPartitionBackup()).thenReturn(6);
    CompletionCallback<StoreBackupId> createCallback = mock(CompletionCallback.class);
    backupAgent.createNewStoreBackup(5, createCallback);
    verify(createCallback, timeout(5000L)).onCompleted(storeBackupId);
    CompletionCallback<Void> verifyCallback = mock(CompletionCallback.class);
    backupAgent.verifyStoreBackup(storeBackupId, verifyCallback);
    verify(mockJnaBackupEngine0, timeout(5000L)).verifyPartitionBackup(7);
    verify(mockJnaBackupEngine1, timeout(5000L)).verifyPartitionBackup(6);
    verify(verifyCallback, timeout(5000L)).onCompleted(null);
    CompletionCallback<Void> clearCallback = mock(CompletionCallback.class);
    backupAgent.clearUnavailableStoreBackups(readyPartitionBackupIds, clearCallback);
    verify(mockJnaBackupEngine0, timeout(5000L)).partitionBackupGc(Arrays.asList(2, 4, 6));
    verify(mockJnaBackupEngine1, timeout(5000L)).partitionBackupGc(Arrays.asList(3, 5, 7));
    verify(clearCallback, timeout(5000L)).onCompleted(null);
    CompletionCallback<Void> restoreCallback = mock(CompletionCallback.class);
    backupAgent.restoreFromStoreBackup(storeBackupId, "restore_root", restoreCallback);
    verify(mockJnaBackupEngine0, timeout(5000L)).restoreFromPartitionBackup(7, Paths.get("restore_root", "0").toString());
    verify(mockJnaBackupEngine1, timeout(5000L)).restoreFromPartitionBackup(6, Paths.get("restore_root", "1").toString());
    verify(restoreCallback, timeout(5000L)).onCompleted(null);
    backupAgent.stop();
}