Search in sources :

Example 1 with BatchSender

use of com.alibaba.graphscope.groot.ingestor.BatchSender in project GraphScope by alibaba.

the class BatchSenderTest method testSend.

@Test
void testSend() throws InterruptedException {
    Configs configs = Configs.newBuilder().put(CommonConfig.STORE_NODE_COUNT.getKey(), "1").build();
    MetaService mockMetaService = mock(MetaService.class);
    when(mockMetaService.getPartitionCount()).thenReturn(2);
    when(mockMetaService.getStoreIdByPartition(anyInt())).thenReturn(0);
    StoreWriter mockStoreWriter = mock(StoreWriter.class);
    String requestId = "test_batch_sender";
    int queueId = 0;
    long snapshotId = 10L;
    long offset = 50L;
    LabelId labelId = new LabelId(0);
    OperationBlob writeVertexBlob1 = new OverwriteVertexOperation(new VertexId(0L), labelId, Collections.EMPTY_MAP).toBlob();
    OperationBlob writeVertexBlob2 = new OverwriteVertexOperation(new VertexId(1L), labelId, Collections.EMPTY_MAP).toBlob();
    OperationBatch batch = OperationBatch.newBuilder().addOperationBlob(writeVertexBlob1).addOperationBlob(writeVertexBlob2).addOperationBlob(OperationBlob.MARKER_OPERATION_BLOB).build();
    CountDownLatch latch = new CountDownLatch(1);
    doAnswer(invocationOnMock -> {
        StoreDataBatch storeBatch = invocationOnMock.getArgument(1);
        CompletionCallback callback = invocationOnMock.getArgument(2);
        assertAll(() -> assertEquals(storeBatch.getRequestId(), requestId), () -> assertEquals(storeBatch.getQueueId(), queueId), () -> assertEquals(storeBatch.getSnapshotId(), snapshotId), () -> assertEquals(storeBatch.getOffset(), offset), () -> assertEquals(storeBatch.getDataBatch().size(), 2));
        List<Map<Integer, OperationBatch>> dataBatch = storeBatch.getDataBatch();
        Map<Integer, OperationBatch> partitionToBatch = dataBatch.get(0);
        assertAll(() -> assertEquals(partitionToBatch.get(0).getOperationBlob(0), writeVertexBlob1), () -> assertEquals(partitionToBatch.get(1).getOperationBlob(0), writeVertexBlob2));
        assertEquals(dataBatch.get(1).get(-1).getOperationBlob(0), OperationBlob.MARKER_OPERATION_BLOB);
        callback.onCompleted(0);
        latch.countDown();
        return null;
    }).when(mockStoreWriter).write(anyInt(), any(), any());
    BatchSender batchSender = new BatchSender(configs, mockMetaService, mockStoreWriter, new MetricsCollector(configs));
    batchSender.start();
    batchSender.asyncSendWithRetry(requestId, queueId, snapshotId, offset, batch);
    assertTrue(latch.await(5L, TimeUnit.SECONDS));
    batchSender.stop();
}
Also used : MetricsCollector(com.alibaba.graphscope.groot.metrics.MetricsCollector) BatchSender(com.alibaba.graphscope.groot.ingestor.BatchSender) MetaService(com.alibaba.graphscope.groot.meta.MetaService) StoreDataBatch(com.alibaba.graphscope.groot.operation.StoreDataBatch) VertexId(com.alibaba.graphscope.groot.operation.VertexId) CountDownLatch(java.util.concurrent.CountDownLatch) OperationBatch(com.alibaba.graphscope.groot.operation.OperationBatch) CompletionCallback(com.alibaba.graphscope.groot.CompletionCallback) StoreWriter(com.alibaba.graphscope.groot.ingestor.StoreWriter) OperationBlob(com.alibaba.graphscope.groot.operation.OperationBlob) OverwriteVertexOperation(com.alibaba.graphscope.groot.operation.dml.OverwriteVertexOperation) Configs(com.alibaba.maxgraph.common.config.Configs) LabelId(com.alibaba.maxgraph.sdkcommon.schema.LabelId) Map(java.util.Map) Test(org.junit.jupiter.api.Test)

Example 2 with BatchSender

use of com.alibaba.graphscope.groot.ingestor.BatchSender in project GraphScope by alibaba.

the class IngestProcessorTest method testIngestProcessor.

@Test
void testIngestProcessor() throws IOException {
    long tailOffset = 50L;
    int queueId = 0;
    Configs configs = Configs.newBuilder().build();
    LogService mockLogService = mock(LogService.class);
    LogReader mockLogReader = mock(LogReader.class);
    when(mockLogService.createReader(queueId, tailOffset + 1)).thenReturn(mockLogReader);
    LogWriter mockLogWriter = mock(LogWriter.class);
    when(mockLogService.createWriter(queueId)).thenReturn(mockLogWriter);
    when(mockLogWriter.append(any())).thenReturn(tailOffset + 3);
    OperationBatch emptyBatch = OperationBatch.newBuilder().build();
    long readSnapshotId = 5L;
    ReadLogEntry readLogEntry1 = new ReadLogEntry(tailOffset + 1, readSnapshotId, emptyBatch);
    ReadLogEntry readLogEntry2 = new ReadLogEntry(tailOffset + 2, readSnapshotId, emptyBatch);
    when(mockLogReader.readNext()).thenReturn(readLogEntry1).thenReturn(readLogEntry2).thenReturn(null);
    BatchSender mockBatchSender = mock(BatchSender.class);
    AtomicLong ingestSnapshotId = new AtomicLong(10L);
    IngestProcessor ingestProcessor = new IngestProcessor(configs, mockLogService, mockBatchSender, queueId, ingestSnapshotId, new MetricsCollector(configs));
    ingestProcessor.setTailOffset(tailOffset);
    ingestProcessor.start();
    verify(mockBatchSender, timeout(5000L)).asyncSendWithRetry(eq(""), eq(queueId), eq(readSnapshotId), eq(tailOffset + 1), any());
    verify(mockBatchSender, timeout(5000L)).asyncSendWithRetry(eq(""), eq(queueId), eq(readSnapshotId), eq(tailOffset + 2), any());
    verify(mockLogReader).close();
    String requestId = "test_ingest_processor";
    IngestCallback mockIngestCallback = mock(IngestCallback.class);
    ingestProcessor.ingestBatch(requestId, emptyBatch, mockIngestCallback);
    verify(mockBatchSender, timeout(5000L)).asyncSendWithRetry(requestId, queueId, ingestSnapshotId.get(), tailOffset + 3, emptyBatch);
    verify(mockIngestCallback, timeout(5000L)).onSuccess(ingestSnapshotId.get());
    ingestProcessor.stop();
    verify(mockLogWriter, timeout(5000L)).close();
}
Also used : MetricsCollector(com.alibaba.graphscope.groot.metrics.MetricsCollector) BatchSender(com.alibaba.graphscope.groot.ingestor.BatchSender) ReadLogEntry(com.alibaba.graphscope.groot.wal.ReadLogEntry) IngestProcessor(com.alibaba.graphscope.groot.ingestor.IngestProcessor) OperationBatch(com.alibaba.graphscope.groot.operation.OperationBatch) AtomicLong(java.util.concurrent.atomic.AtomicLong) LogWriter(com.alibaba.graphscope.groot.wal.LogWriter) Configs(com.alibaba.maxgraph.common.config.Configs) LogReader(com.alibaba.graphscope.groot.wal.LogReader) IngestCallback(com.alibaba.graphscope.groot.ingestor.IngestCallback) LogService(com.alibaba.graphscope.groot.wal.LogService) Test(org.junit.jupiter.api.Test)

Aggregations

BatchSender (com.alibaba.graphscope.groot.ingestor.BatchSender)2 MetricsCollector (com.alibaba.graphscope.groot.metrics.MetricsCollector)2 OperationBatch (com.alibaba.graphscope.groot.operation.OperationBatch)2 Configs (com.alibaba.maxgraph.common.config.Configs)2 Test (org.junit.jupiter.api.Test)2 CompletionCallback (com.alibaba.graphscope.groot.CompletionCallback)1 IngestCallback (com.alibaba.graphscope.groot.ingestor.IngestCallback)1 IngestProcessor (com.alibaba.graphscope.groot.ingestor.IngestProcessor)1 StoreWriter (com.alibaba.graphscope.groot.ingestor.StoreWriter)1 MetaService (com.alibaba.graphscope.groot.meta.MetaService)1 OperationBlob (com.alibaba.graphscope.groot.operation.OperationBlob)1 StoreDataBatch (com.alibaba.graphscope.groot.operation.StoreDataBatch)1 VertexId (com.alibaba.graphscope.groot.operation.VertexId)1 OverwriteVertexOperation (com.alibaba.graphscope.groot.operation.dml.OverwriteVertexOperation)1 LogReader (com.alibaba.graphscope.groot.wal.LogReader)1 LogService (com.alibaba.graphscope.groot.wal.LogService)1 LogWriter (com.alibaba.graphscope.groot.wal.LogWriter)1 ReadLogEntry (com.alibaba.graphscope.groot.wal.ReadLogEntry)1 LabelId (com.alibaba.maxgraph.sdkcommon.schema.LabelId)1 Map (java.util.Map)1