Search in sources :

Example 1 with OperationBlob

use of com.alibaba.graphscope.groot.operation.OperationBlob in project GraphScope by alibaba.

the class BatchSender method asyncSendWithRetry.

public void asyncSendWithRetry(String requestId, int queueId, long snapshotId, long offset, OperationBatch operationBatch) {
    int partitionCount = metaService.getPartitionCount();
    Map<Integer, Builder> storeToBatchBuilder = new HashMap<>();
    Function<Integer, Builder> storeDataBatchBuilderFunc = k -> StoreDataBatch.newBuilder().requestId(requestId).queueId(queueId).snapshotId(snapshotId).offset(offset);
    for (OperationBlob operationBlob : operationBatch) {
        long partitionKey = operationBlob.getPartitionKey();
        if (partitionKey == -1L) {
            // replicate to all store node
            for (int i = 0; i < this.storeCount; i++) {
                StoreDataBatch.Builder batchBuilder = storeToBatchBuilder.computeIfAbsent(i, storeDataBatchBuilderFunc);
                batchBuilder.addOperation(-1, operationBlob);
            }
        } else {
            int partitionId = PartitionUtils.getPartitionIdFromKey(partitionKey, partitionCount);
            int storeId = metaService.getStoreIdByPartition(partitionId);
            StoreDataBatch.Builder batchBuilder = storeToBatchBuilder.computeIfAbsent(storeId, storeDataBatchBuilderFunc);
            batchBuilder.addOperation(partitionId, operationBlob);
        }
    }
    storeToBatchBuilder.forEach((storeId, batchBuilder) -> {
        while (!shouldStop) {
            try {
                storeSendBuffer.get(storeId).put(batchBuilder.build());
                break;
            } catch (InterruptedException e) {
                logger.warn("send buffer interrupted", e);
            }
        }
    });
}
Also used : Configs(com.alibaba.maxgraph.common.config.Configs) CommonConfig(com.alibaba.maxgraph.common.config.CommonConfig) LoggerFactory(org.slf4j.LoggerFactory) IngestorConfig(com.alibaba.maxgraph.common.config.IngestorConfig) OperationBatch(com.alibaba.graphscope.groot.operation.OperationBatch) StoreDataBatch(com.alibaba.graphscope.groot.operation.StoreDataBatch) HashMap(java.util.HashMap) Function(java.util.function.Function) ArrayList(java.util.ArrayList) MetricsCollector(com.alibaba.graphscope.groot.metrics.MetricsCollector) MetricsAgent(com.alibaba.graphscope.groot.metrics.MetricsAgent) Map(java.util.Map) Logger(org.slf4j.Logger) CompletionCallback(com.alibaba.graphscope.groot.CompletionCallback) BlockingQueue(java.util.concurrent.BlockingQueue) MetaService(com.alibaba.graphscope.groot.meta.MetaService) Collectors(java.util.stream.Collectors) TimeUnit(java.util.concurrent.TimeUnit) PartitionUtils(com.alibaba.maxgraph.sdkcommon.util.PartitionUtils) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) StoreConfig(com.alibaba.maxgraph.common.config.StoreConfig) List(java.util.List) AvgMetric(com.alibaba.graphscope.groot.metrics.AvgMetric) Builder(com.alibaba.graphscope.groot.operation.StoreDataBatch.Builder) OperationBlob(com.alibaba.graphscope.groot.operation.OperationBlob) Builder(com.alibaba.graphscope.groot.operation.StoreDataBatch.Builder) HashMap(java.util.HashMap) OperationBlob(com.alibaba.graphscope.groot.operation.OperationBlob) Builder(com.alibaba.graphscope.groot.operation.StoreDataBatch.Builder) StoreDataBatch(com.alibaba.graphscope.groot.operation.StoreDataBatch)

Example 2 with OperationBlob

use of com.alibaba.graphscope.groot.operation.OperationBlob in project GraphScope by alibaba.

the class BatchSenderTest method testSend.

@Test
void testSend() throws InterruptedException {
    Configs configs = Configs.newBuilder().put(CommonConfig.STORE_NODE_COUNT.getKey(), "1").build();
    MetaService mockMetaService = mock(MetaService.class);
    when(mockMetaService.getPartitionCount()).thenReturn(2);
    when(mockMetaService.getStoreIdByPartition(anyInt())).thenReturn(0);
    StoreWriter mockStoreWriter = mock(StoreWriter.class);
    String requestId = "test_batch_sender";
    int queueId = 0;
    long snapshotId = 10L;
    long offset = 50L;
    LabelId labelId = new LabelId(0);
    OperationBlob writeVertexBlob1 = new OverwriteVertexOperation(new VertexId(0L), labelId, Collections.EMPTY_MAP).toBlob();
    OperationBlob writeVertexBlob2 = new OverwriteVertexOperation(new VertexId(1L), labelId, Collections.EMPTY_MAP).toBlob();
    OperationBatch batch = OperationBatch.newBuilder().addOperationBlob(writeVertexBlob1).addOperationBlob(writeVertexBlob2).addOperationBlob(OperationBlob.MARKER_OPERATION_BLOB).build();
    CountDownLatch latch = new CountDownLatch(1);
    doAnswer(invocationOnMock -> {
        StoreDataBatch storeBatch = invocationOnMock.getArgument(1);
        CompletionCallback callback = invocationOnMock.getArgument(2);
        assertAll(() -> assertEquals(storeBatch.getRequestId(), requestId), () -> assertEquals(storeBatch.getQueueId(), queueId), () -> assertEquals(storeBatch.getSnapshotId(), snapshotId), () -> assertEquals(storeBatch.getOffset(), offset), () -> assertEquals(storeBatch.getDataBatch().size(), 2));
        List<Map<Integer, OperationBatch>> dataBatch = storeBatch.getDataBatch();
        Map<Integer, OperationBatch> partitionToBatch = dataBatch.get(0);
        assertAll(() -> assertEquals(partitionToBatch.get(0).getOperationBlob(0), writeVertexBlob1), () -> assertEquals(partitionToBatch.get(1).getOperationBlob(0), writeVertexBlob2));
        assertEquals(dataBatch.get(1).get(-1).getOperationBlob(0), OperationBlob.MARKER_OPERATION_BLOB);
        callback.onCompleted(0);
        latch.countDown();
        return null;
    }).when(mockStoreWriter).write(anyInt(), any(), any());
    BatchSender batchSender = new BatchSender(configs, mockMetaService, mockStoreWriter, new MetricsCollector(configs));
    batchSender.start();
    batchSender.asyncSendWithRetry(requestId, queueId, snapshotId, offset, batch);
    assertTrue(latch.await(5L, TimeUnit.SECONDS));
    batchSender.stop();
}
Also used : MetricsCollector(com.alibaba.graphscope.groot.metrics.MetricsCollector) BatchSender(com.alibaba.graphscope.groot.ingestor.BatchSender) MetaService(com.alibaba.graphscope.groot.meta.MetaService) StoreDataBatch(com.alibaba.graphscope.groot.operation.StoreDataBatch) VertexId(com.alibaba.graphscope.groot.operation.VertexId) CountDownLatch(java.util.concurrent.CountDownLatch) OperationBatch(com.alibaba.graphscope.groot.operation.OperationBatch) CompletionCallback(com.alibaba.graphscope.groot.CompletionCallback) StoreWriter(com.alibaba.graphscope.groot.ingestor.StoreWriter) OperationBlob(com.alibaba.graphscope.groot.operation.OperationBlob) OverwriteVertexOperation(com.alibaba.graphscope.groot.operation.dml.OverwriteVertexOperation) Configs(com.alibaba.maxgraph.common.config.Configs) LabelId(com.alibaba.maxgraph.sdkcommon.schema.LabelId) Map(java.util.Map) Test(org.junit.jupiter.api.Test)

Aggregations

CompletionCallback (com.alibaba.graphscope.groot.CompletionCallback)2 MetaService (com.alibaba.graphscope.groot.meta.MetaService)2 MetricsCollector (com.alibaba.graphscope.groot.metrics.MetricsCollector)2 OperationBatch (com.alibaba.graphscope.groot.operation.OperationBatch)2 OperationBlob (com.alibaba.graphscope.groot.operation.OperationBlob)2 StoreDataBatch (com.alibaba.graphscope.groot.operation.StoreDataBatch)2 Configs (com.alibaba.maxgraph.common.config.Configs)2 Map (java.util.Map)2 BatchSender (com.alibaba.graphscope.groot.ingestor.BatchSender)1 StoreWriter (com.alibaba.graphscope.groot.ingestor.StoreWriter)1 AvgMetric (com.alibaba.graphscope.groot.metrics.AvgMetric)1 MetricsAgent (com.alibaba.graphscope.groot.metrics.MetricsAgent)1 Builder (com.alibaba.graphscope.groot.operation.StoreDataBatch.Builder)1 VertexId (com.alibaba.graphscope.groot.operation.VertexId)1 OverwriteVertexOperation (com.alibaba.graphscope.groot.operation.dml.OverwriteVertexOperation)1 CommonConfig (com.alibaba.maxgraph.common.config.CommonConfig)1 IngestorConfig (com.alibaba.maxgraph.common.config.IngestorConfig)1 StoreConfig (com.alibaba.maxgraph.common.config.StoreConfig)1 LabelId (com.alibaba.maxgraph.sdkcommon.schema.LabelId)1 PartitionUtils (com.alibaba.maxgraph.sdkcommon.util.PartitionUtils)1