Examples with IngestProcessor com.alibaba.graphscope.groot.ingestor.IngestProcessor used on opensource projects

Example 1 with IngestProcessor

use of com.alibaba.graphscope.groot.ingestor.IngestProcessor in project GraphScope by alibaba.

the class IngestProcessorTest method testIngestProcessor.

@Test
void testIngestProcessor() throws IOException {
    long tailOffset = 50L;
    int queueId = 0;
    Configs configs = Configs.newBuilder().build();
    LogService mockLogService = mock(LogService.class);
    LogReader mockLogReader = mock(LogReader.class);
    when(mockLogService.createReader(queueId, tailOffset + 1)).thenReturn(mockLogReader);
    LogWriter mockLogWriter = mock(LogWriter.class);
    when(mockLogService.createWriter(queueId)).thenReturn(mockLogWriter);
    when(mockLogWriter.append(any())).thenReturn(tailOffset + 3);
    OperationBatch emptyBatch = OperationBatch.newBuilder().build();
    long readSnapshotId = 5L;
    ReadLogEntry readLogEntry1 = new ReadLogEntry(tailOffset + 1, readSnapshotId, emptyBatch);
    ReadLogEntry readLogEntry2 = new ReadLogEntry(tailOffset + 2, readSnapshotId, emptyBatch);
    when(mockLogReader.readNext()).thenReturn(readLogEntry1).thenReturn(readLogEntry2).thenReturn(null);
    BatchSender mockBatchSender = mock(BatchSender.class);
    AtomicLong ingestSnapshotId = new AtomicLong(10L);
    IngestProcessor ingestProcessor = new IngestProcessor(configs, mockLogService, mockBatchSender, queueId, ingestSnapshotId, new MetricsCollector(configs));
    ingestProcessor.setTailOffset(tailOffset);
    ingestProcessor.start();
    verify(mockBatchSender, timeout(5000L)).asyncSendWithRetry(eq(""), eq(queueId), eq(readSnapshotId), eq(tailOffset + 1), any());
    verify(mockBatchSender, timeout(5000L)).asyncSendWithRetry(eq(""), eq(queueId), eq(readSnapshotId), eq(tailOffset + 2), any());
    verify(mockLogReader).close();
    String requestId = "test_ingest_processor";
    IngestCallback mockIngestCallback = mock(IngestCallback.class);
    ingestProcessor.ingestBatch(requestId, emptyBatch, mockIngestCallback);
    verify(mockBatchSender, timeout(5000L)).asyncSendWithRetry(requestId, queueId, ingestSnapshotId.get(), tailOffset + 3, emptyBatch);
    verify(mockIngestCallback, timeout(5000L)).onSuccess(ingestSnapshotId.get());
    ingestProcessor.stop();
    verify(mockLogWriter, timeout(5000L)).close();
}

Example 2 with IngestProcessor

use of com.alibaba.graphscope.groot.ingestor.IngestProcessor in project GraphScope by alibaba.

the class IngestServiceTest method testIngestService.

@Test
void testIngestService() {
    Configs configs = Configs.newBuilder().put(CommonConfig.NODE_IDX.getKey(), "0").put(CommonConfig.STORE_NODE_COUNT.getKey(), "1").put(IngestorConfig.INGESTOR_CHECK_PROCESSOR_INTERVAL_MS.getKey(), "100").build();
    MockDiscovery mockDiscovery = new MockDiscovery();
    MetaService mockMetaService = mock(MetaService.class);
    when(mockMetaService.getQueueIdsForIngestor(0)).thenReturn(Arrays.asList(0));
    LogService mockLogService = mock(LogService.class);
    IngestProgressFetcher mockIngestProgressFetcher = mock(IngestProgressFetcher.class);
    when(mockIngestProgressFetcher.getTailOffsets(Arrays.asList(0))).thenReturn(Arrays.asList(50L));
    StoreWriter mockStoreWriter = mock(StoreWriter.class);
    IngestService spyIngestService = spy(new IngestService(configs, mockDiscovery, mockMetaService, mockLogService, mockIngestProgressFetcher, mockStoreWriter, new MetricsCollector(configs)));
    IngestProcessor mockIngestProcessor = mock(IngestProcessor.class);
    doReturn(mockIngestProcessor).when(spyIngestService).makeIngestProcessor(any(), any(), any(), eq(0), any(), any());
    spyIngestService.start();
    verify(mockIngestProcessor, never()).start();
    mockDiscovery.addNode(RoleType.STORE, Collections.singletonMap(0, null));
    verify(mockIngestProcessor, timeout(5000L)).setTailOffset(50L);
    verify(mockIngestProcessor, timeout(5000L)).start();
    spyIngestService.advanceIngestSnapshotId(5L, null);
    verify(mockIngestProcessor).ingestBatch(eq("marker"), eq(IngestService.MARKER_BATCH), any());
    mockDiscovery.removeNode(RoleType.STORE, Collections.singletonMap(0, null));
    verify(mockIngestProcessor, timeout(5000L).times(2)).stop();
    spyIngestService.stop();
}