Examples with CompletionCallback com.alibaba.graphscope.groot.CompletionCallback used on opensource projects

Example 6 with CompletionCallback

use of com.alibaba.graphscope.groot.CompletionCallback in project GraphScope by alibaba.

the class BatchSenderTest method testSend.

@Test
void testSend() throws InterruptedException {
    Configs configs = Configs.newBuilder().put(CommonConfig.STORE_NODE_COUNT.getKey(), "1").build();
    MetaService mockMetaService = mock(MetaService.class);
    when(mockMetaService.getPartitionCount()).thenReturn(2);
    when(mockMetaService.getStoreIdByPartition(anyInt())).thenReturn(0);
    StoreWriter mockStoreWriter = mock(StoreWriter.class);
    String requestId = "test_batch_sender";
    int queueId = 0;
    long snapshotId = 10L;
    long offset = 50L;
    LabelId labelId = new LabelId(0);
    OperationBlob writeVertexBlob1 = new OverwriteVertexOperation(new VertexId(0L), labelId, Collections.EMPTY_MAP).toBlob();
    OperationBlob writeVertexBlob2 = new OverwriteVertexOperation(new VertexId(1L), labelId, Collections.EMPTY_MAP).toBlob();
    OperationBatch batch = OperationBatch.newBuilder().addOperationBlob(writeVertexBlob1).addOperationBlob(writeVertexBlob2).addOperationBlob(OperationBlob.MARKER_OPERATION_BLOB).build();
    CountDownLatch latch = new CountDownLatch(1);
    doAnswer(invocationOnMock -> {
        StoreDataBatch storeBatch = invocationOnMock.getArgument(1);
        CompletionCallback callback = invocationOnMock.getArgument(2);
        assertAll(() -> assertEquals(storeBatch.getRequestId(), requestId), () -> assertEquals(storeBatch.getQueueId(), queueId), () -> assertEquals(storeBatch.getSnapshotId(), snapshotId), () -> assertEquals(storeBatch.getOffset(), offset), () -> assertEquals(storeBatch.getDataBatch().size(), 2));
        List<Map<Integer, OperationBatch>> dataBatch = storeBatch.getDataBatch();
        Map<Integer, OperationBatch> partitionToBatch = dataBatch.get(0);
        assertAll(() -> assertEquals(partitionToBatch.get(0).getOperationBlob(0), writeVertexBlob1), () -> assertEquals(partitionToBatch.get(1).getOperationBlob(0), writeVertexBlob2));
        assertEquals(dataBatch.get(1).get(-1).getOperationBlob(0), OperationBlob.MARKER_OPERATION_BLOB);
        callback.onCompleted(0);
        latch.countDown();
        return null;
    }).when(mockStoreWriter).write(anyInt(), any(), any());
    BatchSender batchSender = new BatchSender(configs, mockMetaService, mockStoreWriter, new MetricsCollector(configs));
    batchSender.start();
    batchSender.asyncSendWithRetry(requestId, queueId, snapshotId, offset, batch);
    assertTrue(latch.await(5L, TimeUnit.SECONDS));
    batchSender.stop();
}

Example 7 with CompletionCallback

use of com.alibaba.graphscope.groot.CompletionCallback in project GraphScope by alibaba.

the class NotifyFrontendListener method snapshotAdvanced.

@Override
public void snapshotAdvanced(long snapshotId, long ddlSnapshotId) {
    logger.debug("snapshot advance to [" + snapshotId + "]-[" + ddlSnapshotId + "], will notify frontend");
    GraphDef graphDef = null;
    if (ddlSnapshotId > this.lastDdlSnapshotId.get()) {
        graphDef = this.schemaManager.getGraphDef();
    }
    this.frontendSnapshotClient.advanceQuerySnapshot(snapshotId, graphDef, new CompletionCallback<Long>() {

        @Override
        public void onCompleted(Long res) {
            if (res >= snapshotId) {
                logger.warn("unexpected previousSnapshotId [" + res + "], should <= [" + snapshotId + "]. frontend [" + frontendId + "]");
            } else {
                lastDdlSnapshotId.getAndUpdate(x -> x < ddlSnapshotId ? ddlSnapshotId : x);
            }
        }

        @Override
        public void onError(Throwable t) {
            logger.error("error in advanceQuerySnapshot [" + snapshotId + "], ddlSnapshotId [" + ddlSnapshotId + "], frontend [" + frontendId + "]", t);
        }
    });
}

Example 8 with CompletionCallback

use of com.alibaba.graphscope.groot.CompletionCallback in project GraphScope by alibaba.

the class BatchSender method sendBatch.

private void sendBatch() {
    SendTask sendTask;
    try {
        sendTask = this.sendTasks.poll(1000L, TimeUnit.MILLISECONDS);
    } catch (InterruptedException e) {
        logger.warn("polling send task interrupted", e);
        return;
    }
    if (sendTask == null) {
        return;
    }
    int storeId = sendTask.storeId;
    List<StoreDataBatch> dataToSend = sendTask.dataToRetry;
    if (dataToSend == null) {
        dataToSend = new ArrayList<>();
        BlockingQueue<StoreDataBatch> buffer = this.storeSendBuffer.get(storeId);
        StoreDataBatch dataBatch;
        int operationCount = 0;
        int batchCount = 0;
        while (operationCount < this.sendOperationLimit && batchCount < this.receiverQueueSize && (dataBatch = buffer.poll()) != null) {
            dataToSend.add(dataBatch);
            operationCount += dataBatch.getSize();
            batchCount++;
        }
    }
    if (dataToSend.size() > 0) {
        List<StoreDataBatch> finalDataToSend = dataToSend;
        long beforeWriteTime = System.nanoTime();
        this.storeWriter.write(storeId, dataToSend, new CompletionCallback<Integer>() {

            @Override
            public void onCompleted(Integer res) {
                sendBytesMetric.add(res);
                sendRecordsMetric.add(finalDataToSend.stream().collect(Collectors.summingInt(batch -> batch.getSize())));
                finish(true);
            }

            @Override
            public void onError(Throwable t) {
                logger.warn("send to store [" + storeId + "] failed. will retry later", t);
                finish(false);
            }

            private void finish(boolean suc) {
                long finishTime = System.nanoTime();
                callbackLatencyMetrics.get(storeId).add(finishTime - beforeWriteTime);
                if (suc) {
                    addTask(storeId, null);
                } else {
                    addTask(storeId, finalDataToSend);
                }
            }
        });
    } else {
        addTask(storeId, null);
    }
}