Search in sources :

Example 1 with SnapshotManager

use of com.alibaba.graphscope.groot.coordinator.SnapshotManager in project GraphScope by alibaba.

the class SnapshotNotifierTest method testSnapshotNotifier.

@Test
void testSnapshotNotifier() {
    NodeDiscovery discovery = mock(NodeDiscovery.class);
    SnapshotManager snapshotManager = mock(SnapshotManager.class);
    RoleClients<FrontendSnapshotClient> roleClients = mock(RoleClients.class);
    SnapshotNotifier snapshotNotifier = new SnapshotNotifier(discovery, snapshotManager, null, roleClients);
    snapshotNotifier.start();
    MaxGraphNode localNode = MaxGraphNode.createLocalNode(Configs.newBuilder().put(CommonConfig.ROLE_NAME.getKey(), RoleType.COORDINATOR.getName()).build(), 1111);
    snapshotNotifier.nodesJoin(RoleType.FRONTEND, Collections.singletonMap(1, localNode));
    ArgumentCaptor<NotifyFrontendListener> captor = ArgumentCaptor.forClass(NotifyFrontendListener.class);
    verify(snapshotManager).addListener(captor.capture());
    NotifyFrontendListener listener = captor.getValue();
    snapshotNotifier.nodesLeft(RoleType.FRONTEND, Collections.singletonMap(1, localNode));
    verify(snapshotManager).removeListener(listener);
}
Also used : SnapshotNotifier(com.alibaba.graphscope.groot.coordinator.SnapshotNotifier) MaxGraphNode(com.alibaba.graphscope.groot.discovery.MaxGraphNode) NotifyFrontendListener(com.alibaba.graphscope.groot.coordinator.NotifyFrontendListener) NodeDiscovery(com.alibaba.graphscope.groot.discovery.NodeDiscovery) FrontendSnapshotClient(com.alibaba.graphscope.groot.coordinator.FrontendSnapshotClient) SnapshotManager(com.alibaba.graphscope.groot.coordinator.SnapshotManager) Test(org.junit.jupiter.api.Test)

Example 2 with SnapshotManager

use of com.alibaba.graphscope.groot.coordinator.SnapshotManager in project GraphScope by alibaba.

the class IngestProgressServiceTest method testIngestProgressService.

@Test
void testIngestProgressService() {
    SnapshotManager snapshotManager = mock(SnapshotManager.class);
    when(snapshotManager.getTailOffsets(Arrays.asList(1))).thenReturn(Arrays.asList(10L));
    IngestProgressService ingestProgressService = new IngestProgressService(snapshotManager);
    GetTailOffsetsRequest request = GetTailOffsetsRequest.newBuilder().addQueueId(1).build();
    ingestProgressService.getTailOffsets(request, new StreamObserver<GetTailOffsetsResponse>() {

        @Override
        public void onNext(GetTailOffsetsResponse response) {
            List<Long> offsetsList = response.getOffsetsList();
            assertEquals(offsetsList.size(), 1);
            assertEquals(offsetsList.get(0), 10L);
        }

        @Override
        public void onError(Throwable t) {
            throw new RuntimeException(t);
        }

        @Override
        public void onCompleted() {
        }
    });
    verify(snapshotManager).getTailOffsets(Arrays.asList(1));
}
Also used : List(java.util.List) GetTailOffsetsRequest(com.alibaba.maxgraph.proto.groot.GetTailOffsetsRequest) GetTailOffsetsResponse(com.alibaba.maxgraph.proto.groot.GetTailOffsetsResponse) SnapshotManager(com.alibaba.graphscope.groot.coordinator.SnapshotManager) IngestProgressService(com.alibaba.graphscope.groot.coordinator.IngestProgressService) Test(org.junit.jupiter.api.Test)

Example 3 with SnapshotManager

use of com.alibaba.graphscope.groot.coordinator.SnapshotManager in project GraphScope by alibaba.

the class SchemaManagerTest method testSchemaManager.

@Test
void testSchemaManager() throws IOException, InterruptedException {
    SnapshotManager mockSnapshotManager = mock(SnapshotManager.class);
    doAnswer(invocationOnMock -> {
        SnapshotListener listener = invocationOnMock.getArgument(1);
        listener.onSnapshotAvailable();
        return null;
    }).when(mockSnapshotManager).addSnapshotListener(anyLong(), any());
    when(mockSnapshotManager.increaseWriteSnapshotId()).thenReturn(1L);
    when(mockSnapshotManager.getCurrentWriteSnapshotId()).thenReturn(1L);
    DdlExecutors ddlExecutors = new DdlExecutors();
    DdlWriter mockDdlWriter = mock(DdlWriter.class);
    when(mockDdlWriter.writeOperations(anyString(), any())).thenReturn(new BatchId(1L));
    MetaService mockMetaService = mock(MetaService.class);
    GraphDefFetcher mockGraphDefFetcher = mock(GraphDefFetcher.class);
    GraphDef initialGraphDef = GraphDef.newBuilder().build();
    when(mockGraphDefFetcher.fetchGraphDef()).thenReturn(initialGraphDef);
    SchemaManager schemaManager = new SchemaManager(mockSnapshotManager, ddlExecutors, mockDdlWriter, mockMetaService, mockGraphDefFetcher);
    schemaManager.start();
    assertEquals(initialGraphDef, schemaManager.getGraphDef());
    PropertyValue defaultValue = new PropertyValue(DataType.INT, ByteBuffer.allocate(Integer.BYTES).putInt(1).array());
    PropertyDef propertyDef = new PropertyDef(1, 1, "p1", DataType.INT, defaultValue, true, "property_1");
    TypeDef typeDef = TypeDef.newBuilder().setLabel("vertex1").addPropertyDef(propertyDef).setTypeEnum(TypeEnum.VERTEX).build();
    DdlRequestBatch ddlRequestBatch = DdlRequestBatch.newBuilder().addDdlRequest(new CreateVertexTypeRequest(typeDef)).build();
    CountDownLatch latch = new CountDownLatch(1);
    schemaManager.submitBatchDdl("requestId", "sessionId", ddlRequestBatch, new CompletionCallback<Long>() {

        @Override
        public void onCompleted(Long res) {
            latch.countDown();
        }

        @Override
        public void onError(Throwable t) {
        }
    });
    assertTrue(latch.await(5L, TimeUnit.SECONDS));
    schemaManager.stop();
}
Also used : MetaService(com.alibaba.graphscope.groot.meta.MetaService) PropertyDef(com.alibaba.maxgraph.sdkcommon.schema.PropertyDef) BatchId(com.alibaba.graphscope.groot.operation.BatchId) PropertyValue(com.alibaba.maxgraph.sdkcommon.schema.PropertyValue) SchemaManager(com.alibaba.graphscope.groot.coordinator.SchemaManager) CountDownLatch(java.util.concurrent.CountDownLatch) GraphDefFetcher(com.alibaba.graphscope.groot.coordinator.GraphDefFetcher) GraphDef(com.alibaba.maxgraph.sdkcommon.schema.GraphDef) SnapshotListener(com.alibaba.graphscope.groot.SnapshotListener) DdlWriter(com.alibaba.graphscope.groot.coordinator.DdlWriter) TypeDef(com.alibaba.maxgraph.sdkcommon.schema.TypeDef) DdlExecutors(com.alibaba.graphscope.groot.schema.ddl.DdlExecutors) ArgumentMatchers.anyLong(org.mockito.ArgumentMatchers.anyLong) CreateVertexTypeRequest(com.alibaba.graphscope.groot.schema.request.CreateVertexTypeRequest) DdlRequestBatch(com.alibaba.graphscope.groot.schema.request.DdlRequestBatch) SnapshotManager(com.alibaba.graphscope.groot.coordinator.SnapshotManager) Test(org.junit.jupiter.api.Test)

Example 4 with SnapshotManager

use of com.alibaba.graphscope.groot.coordinator.SnapshotManager in project GraphScope by alibaba.

the class SnapshotManagerTest method testSnapshotManager.

@Test
void testSnapshotManager() throws IOException, InterruptedException {
    Configs configs = Configs.newBuilder().put(CommonConfig.INGESTOR_QUEUE_COUNT.getKey(), "1").put(CommonConfig.STORE_NODE_COUNT.getKey(), "2").put(CommonConfig.FRONTEND_NODE_COUNT.getKey(), "1").put(CommonConfig.INGESTOR_NODE_COUNT.getKey(), "1").put(CoordinatorConfig.SNAPSHOT_INCREASE_INTERVAL_MS.getKey(), "1000").put(CoordinatorConfig.OFFSETS_PERSIST_INTERVAL_MS.getKey(), "1000").build();
    long querySnapshotId = 10L;
    long writeSnapshotId = 12L;
    List<Long> queueOffsets = Arrays.asList(50L);
    long commitSnapshotId1 = 11L;
    long commitSnapshotId2 = 12L;
    List<Long> commitQueueOffsets1 = Arrays.asList(60L);
    List<Long> commitQueueOffsets2 = Arrays.asList(70L);
    ObjectMapper objectMapper = new ObjectMapper();
    MetaStore mockMetaStore = mock(MetaStore.class);
    when(mockMetaStore.exists(anyString())).thenReturn(true);
    when(mockMetaStore.read(QUERY_SNAPSHOT_INFO_PATH)).thenReturn(objectMapper.writeValueAsBytes(new SnapshotInfo(querySnapshotId, querySnapshotId)));
    when(mockMetaStore.read(WRITE_SNAPSHOT_ID_PATH)).thenReturn(objectMapper.writeValueAsBytes(writeSnapshotId));
    when(mockMetaStore.read(QUEUE_OFFSETS_PATH)).thenReturn(objectMapper.writeValueAsBytes(queueOffsets));
    CountDownLatch updateWriteSnapshotLatch = new CountDownLatch(1);
    doAnswer(invocationOnMock -> {
        updateWriteSnapshotLatch.countDown();
        return null;
    }).when(mockMetaStore).write(WRITE_SNAPSHOT_ID_PATH, objectMapper.writeValueAsBytes(writeSnapshotId + 1));
    CountDownLatch updateQueueOffsetLatch = new CountDownLatch(1);
    doAnswer(invocationOnMock -> {
        updateQueueOffsetLatch.countDown();
        return null;
    }).when(mockMetaStore).write(QUEUE_OFFSETS_PATH, objectMapper.writeValueAsBytes(commitQueueOffsets1));
    IngestorWriteSnapshotIdNotifier mockWriteSnapshotIdNotifier = mock(IngestorWriteSnapshotIdNotifier.class);
    CountDownLatch updateIngestorLatch = new CountDownLatch(1);
    doAnswer(invocationOnMock -> {
        updateIngestorLatch.countDown();
        return null;
    }).when(mockWriteSnapshotIdNotifier).notifyWriteSnapshotIdChanged(writeSnapshotId + 1);
    LogService mockLogService = mock(LogService.class);
    SnapshotManager snapshotManager = new SnapshotManager(configs, mockMetaStore, mockLogService, mockWriteSnapshotIdNotifier);
    snapshotManager.start();
    assertEquals(snapshotManager.getQueueOffsets(), queueOffsets);
    assertTrue(updateWriteSnapshotLatch.await(5L, TimeUnit.SECONDS));
    assertTrue(updateIngestorLatch.await(5L, TimeUnit.SECONDS));
    snapshotManager.commitSnapshotId(0, commitSnapshotId1, 10L, commitQueueOffsets1);
    snapshotManager.commitSnapshotId(1, commitSnapshotId2, 10L, commitQueueOffsets2);
    verify(mockMetaStore).write(QUERY_SNAPSHOT_INFO_PATH, objectMapper.writeValueAsBytes(new SnapshotInfo(commitSnapshotId1, 10L)));
    assertTrue(updateQueueOffsetLatch.await(5L, TimeUnit.SECONDS));
    assertEquals(snapshotManager.getQueueOffsets(), commitQueueOffsets1);
    snapshotManager.stop();
}
Also used : MetaStore(com.alibaba.graphscope.groot.meta.MetaStore) SnapshotInfo(com.alibaba.graphscope.groot.coordinator.SnapshotInfo) Configs(com.alibaba.maxgraph.common.config.Configs) CountDownLatch(java.util.concurrent.CountDownLatch) IngestorWriteSnapshotIdNotifier(com.alibaba.graphscope.groot.coordinator.IngestorWriteSnapshotIdNotifier) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) LogService(com.alibaba.graphscope.groot.wal.LogService) SnapshotManager(com.alibaba.graphscope.groot.coordinator.SnapshotManager) Test(org.junit.jupiter.api.Test)

Example 5 with SnapshotManager

use of com.alibaba.graphscope.groot.coordinator.SnapshotManager in project GraphScope by alibaba.

the class LogRecyclerTest method testRecycler.

@Test
void testRecycler() throws IOException {
    Configs configs = Configs.newBuilder().put(CoordinatorConfig.LOG_RECYCLE_INTERVAL_SECOND.getKey(), "1").build();
    SnapshotManager mockSnapshotManager = mock(SnapshotManager.class);
    when(mockSnapshotManager.getQueueOffsets()).thenReturn(Arrays.asList(1L, 2L, 3L));
    LogService mockLogService = mock(LogService.class);
    LogRecycler logRecycler = new LogRecycler(configs, mockLogService, mockSnapshotManager);
    CountDownLatch latch1 = new CountDownLatch(1);
    CountDownLatch latch2 = new CountDownLatch(1);
    CountDownLatch latch3 = new CountDownLatch(1);
    doAnswer(invocationOnMock -> {
        latch1.countDown();
        return null;
    }).when(mockLogService).deleteBeforeOffset(0, 1L);
    doAnswer(invocationOnMock -> {
        latch2.countDown();
        return null;
    }).when(mockLogService).deleteBeforeOffset(1, 2L);
    doAnswer(invocationOnMock -> {
        latch3.countDown();
        return null;
    }).when(mockLogService).deleteBeforeOffset(2, 3L);
    logRecycler.start();
    assertAll(() -> assertTrue(latch1.await(5L, TimeUnit.SECONDS)), () -> assertTrue(latch2.await(5L, TimeUnit.SECONDS)), () -> assertTrue(latch3.await(5L, TimeUnit.SECONDS)));
    logRecycler.stop();
}
Also used : LogRecycler(com.alibaba.graphscope.groot.coordinator.LogRecycler) Configs(com.alibaba.maxgraph.common.config.Configs) CountDownLatch(java.util.concurrent.CountDownLatch) LogService(com.alibaba.graphscope.groot.wal.LogService) SnapshotManager(com.alibaba.graphscope.groot.coordinator.SnapshotManager) Test(org.junit.jupiter.api.Test)

Aggregations

SnapshotManager (com.alibaba.graphscope.groot.coordinator.SnapshotManager)5 Test (org.junit.jupiter.api.Test)5 CountDownLatch (java.util.concurrent.CountDownLatch)3 LogService (com.alibaba.graphscope.groot.wal.LogService)2 Configs (com.alibaba.maxgraph.common.config.Configs)2 SnapshotListener (com.alibaba.graphscope.groot.SnapshotListener)1 DdlWriter (com.alibaba.graphscope.groot.coordinator.DdlWriter)1 FrontendSnapshotClient (com.alibaba.graphscope.groot.coordinator.FrontendSnapshotClient)1 GraphDefFetcher (com.alibaba.graphscope.groot.coordinator.GraphDefFetcher)1 IngestProgressService (com.alibaba.graphscope.groot.coordinator.IngestProgressService)1 IngestorWriteSnapshotIdNotifier (com.alibaba.graphscope.groot.coordinator.IngestorWriteSnapshotIdNotifier)1 LogRecycler (com.alibaba.graphscope.groot.coordinator.LogRecycler)1 NotifyFrontendListener (com.alibaba.graphscope.groot.coordinator.NotifyFrontendListener)1 SchemaManager (com.alibaba.graphscope.groot.coordinator.SchemaManager)1 SnapshotInfo (com.alibaba.graphscope.groot.coordinator.SnapshotInfo)1 SnapshotNotifier (com.alibaba.graphscope.groot.coordinator.SnapshotNotifier)1 MaxGraphNode (com.alibaba.graphscope.groot.discovery.MaxGraphNode)1 NodeDiscovery (com.alibaba.graphscope.groot.discovery.NodeDiscovery)1 MetaService (com.alibaba.graphscope.groot.meta.MetaService)1 MetaStore (com.alibaba.graphscope.groot.meta.MetaStore)1