Examples with StoreDataBatch com.alibaba.graphscope.groot.operation.StoreDataBatch used on opensource projects

Example 6 with StoreDataBatch

use of com.alibaba.graphscope.groot.operation.StoreDataBatch in project GraphScope by alibaba.

the class WriterAgent method processBatches.

private void processBatches() {
    while (!shouldStop) {
        try {
            long beforePollNano = System.nanoTime();
            StoreDataBatch storeDataBatch = this.bufferQueue.poll();
            long afterPollNano = System.nanoTime();
            long pollNano = afterPollNano - beforePollNano;
            this.totalPollLatencyNano += pollNano;
            this.maxPollLatencyNano.updateAndGet(curMax -> (pollNano > curMax) ? pollNano : curMax);
            if (storeDataBatch == null) {
                continue;
            }
            long batchSnapshotId = storeDataBatch.getSnapshotId();
            logger.debug("polled one batch [" + batchSnapshotId + "]");
            boolean hasDdl = writeEngineWithRetry(storeDataBatch);
            int writeCount = storeDataBatch.getSize();
            this.totalWrite += writeCount;
            if (this.consumeSnapshotId < batchSnapshotId) {
                SnapshotInfo availSnapshotInfo = this.availSnapshotInfoRef.get();
                long availDdlSnapshotId = availSnapshotInfo.getDdlSnapshotId();
                if (availDdlSnapshotId < this.consumeDdlSnapshotId) {
                    availDdlSnapshotId = this.consumeDdlSnapshotId;
                }
                long prevSnapshotId = batchSnapshotId - 1;
                long availSnapshotId = availSnapshotInfo.getSnapshotId();
                if (availSnapshotId < prevSnapshotId) {
                    availSnapshotId = prevSnapshotId;
                }
                this.consumeSnapshotId = batchSnapshotId;
                this.availSnapshotInfoRef.set(new SnapshotInfo(availSnapshotId, availDdlSnapshotId));
                this.commitExecutor.execute(() -> asyncCommit());
            }
            if (hasDdl) {
                this.consumeDdlSnapshotId = batchSnapshotId;
            }
            int queueId = storeDataBatch.getQueueId();
            long offset = storeDataBatch.getOffset();
            this.consumedQueueOffsets.set(queueId, offset);
        } catch (Exception e) {
            logger.error("error in processBatches, ignore", e);
        }
    }
}

Example 7 with StoreDataBatch

use of com.alibaba.graphscope.groot.operation.StoreDataBatch in project GraphScope by alibaba.

the class BatchSenderTest method testSend.

@Test
void testSend() throws InterruptedException {
    Configs configs = Configs.newBuilder().put(CommonConfig.STORE_NODE_COUNT.getKey(), "1").build();
    MetaService mockMetaService = mock(MetaService.class);
    when(mockMetaService.getPartitionCount()).thenReturn(2);
    when(mockMetaService.getStoreIdByPartition(anyInt())).thenReturn(0);
    StoreWriter mockStoreWriter = mock(StoreWriter.class);
    String requestId = "test_batch_sender";
    int queueId = 0;
    long snapshotId = 10L;
    long offset = 50L;
    LabelId labelId = new LabelId(0);
    OperationBlob writeVertexBlob1 = new OverwriteVertexOperation(new VertexId(0L), labelId, Collections.EMPTY_MAP).toBlob();
    OperationBlob writeVertexBlob2 = new OverwriteVertexOperation(new VertexId(1L), labelId, Collections.EMPTY_MAP).toBlob();
    OperationBatch batch = OperationBatch.newBuilder().addOperationBlob(writeVertexBlob1).addOperationBlob(writeVertexBlob2).addOperationBlob(OperationBlob.MARKER_OPERATION_BLOB).build();
    CountDownLatch latch = new CountDownLatch(1);
    doAnswer(invocationOnMock -> {
        StoreDataBatch storeBatch = invocationOnMock.getArgument(1);
        CompletionCallback callback = invocationOnMock.getArgument(2);
        assertAll(() -> assertEquals(storeBatch.getRequestId(), requestId), () -> assertEquals(storeBatch.getQueueId(), queueId), () -> assertEquals(storeBatch.getSnapshotId(), snapshotId), () -> assertEquals(storeBatch.getOffset(), offset), () -> assertEquals(storeBatch.getDataBatch().size(), 2));
        List<Map<Integer, OperationBatch>> dataBatch = storeBatch.getDataBatch();
        Map<Integer, OperationBatch> partitionToBatch = dataBatch.get(0);
        assertAll(() -> assertEquals(partitionToBatch.get(0).getOperationBlob(0), writeVertexBlob1), () -> assertEquals(partitionToBatch.get(1).getOperationBlob(0), writeVertexBlob2));
        assertEquals(dataBatch.get(1).get(-1).getOperationBlob(0), OperationBlob.MARKER_OPERATION_BLOB);
        callback.onCompleted(0);
        latch.countDown();
        return null;
    }).when(mockStoreWriter).write(anyInt(), any(), any());
    BatchSender batchSender = new BatchSender(configs, mockMetaService, mockStoreWriter, new MetricsCollector(configs));
    batchSender.start();
    batchSender.asyncSendWithRetry(requestId, queueId, snapshotId, offset, batch);
    assertTrue(latch.await(5L, TimeUnit.SECONDS));
    batchSender.stop();
}

Example 8 with StoreDataBatch

use of com.alibaba.graphscope.groot.operation.StoreDataBatch in project GraphScope by alibaba.

the class StoreWriteService method writeStore.

@Override
public void writeStore(WriteStoreRequest request, StreamObserver<WriteStoreResponse> responseObserver) {
    List<StoreDataBatchPb> dataBatchesList = request.getDataBatchesList();
    List<StoreDataBatch> batches = new ArrayList<>(dataBatchesList.size());
    try {
        for (StoreDataBatchPb pb : dataBatchesList) {
            batches.add(StoreDataBatch.parseProto(pb));
        }
        boolean success = writerAgent.writeStore2(batches);
        WriteStoreResponse response = WriteStoreResponse.newBuilder().setSuccess(success).build();
        responseObserver.onNext(response);
        responseObserver.onCompleted();
    } catch (Exception e) {
        responseObserver.onError(e);
    }
}

Example 9 with StoreDataBatch

use of com.alibaba.graphscope.groot.operation.StoreDataBatch in project GraphScope by alibaba.

the class BatchSender method sendBatch.

private void sendBatch() {
    SendTask sendTask;
    try {
        sendTask = this.sendTasks.poll(1000L, TimeUnit.MILLISECONDS);
    } catch (InterruptedException e) {
        logger.warn("polling send task interrupted", e);
        return;
    }
    if (sendTask == null) {
        return;
    }
    int storeId = sendTask.storeId;
    List<StoreDataBatch> dataToSend = sendTask.dataToRetry;
    if (dataToSend == null) {
        dataToSend = new ArrayList<>();
        BlockingQueue<StoreDataBatch> buffer = this.storeSendBuffer.get(storeId);
        StoreDataBatch dataBatch;
        int operationCount = 0;
        int batchCount = 0;
        while (operationCount < this.sendOperationLimit && batchCount < this.receiverQueueSize && (dataBatch = buffer.poll()) != null) {
            dataToSend.add(dataBatch);
            operationCount += dataBatch.getSize();
            batchCount++;
        }
    }
    if (dataToSend.size() > 0) {
        List<StoreDataBatch> finalDataToSend = dataToSend;
        long beforeWriteTime = System.nanoTime();
        this.storeWriter.write(storeId, dataToSend, new CompletionCallback<Integer>() {

            @Override
            public void onCompleted(Integer res) {
                sendBytesMetric.add(res);
                sendRecordsMetric.add(finalDataToSend.stream().collect(Collectors.summingInt(batch -> batch.getSize())));
                finish(true);
            }

            @Override
            public void onError(Throwable t) {
                logger.warn("send to store [" + storeId + "] failed. will retry later", t);
                finish(false);
            }

            private void finish(boolean suc) {
                long finishTime = System.nanoTime();
                callbackLatencyMetrics.get(storeId).add(finishTime - beforeWriteTime);
                if (suc) {
                    addTask(storeId, null);
                } else {
                    addTask(storeId, finalDataToSend);
                }
            }
        });
    } else {
        addTask(storeId, null);
    }
}

Example 10 with StoreDataBatch

use of com.alibaba.graphscope.groot.operation.StoreDataBatch in project GraphScope by alibaba.

the class StoreWriteClient method writeStore.

public void writeStore(List<StoreDataBatch> storeDataBatches, CompletionCallback<Integer> callback) {
    Builder builder = WriteStoreRequest.newBuilder();
    for (StoreDataBatch storeDataBatch : storeDataBatches) {
        builder.addDataBatches(storeDataBatch.toProto());
    }
    WriteStoreRequest req = builder.build();
    stub.writeStore(req, new StreamObserver<WriteStoreResponse>() {

        @Override
        public void onNext(WriteStoreResponse writeStoreResponse) {
            boolean success = writeStoreResponse.getSuccess();
            if (success) {
                callback.onCompleted(req.getSerializedSize());
            } else {
                onError(new RuntimeException("store buffer is full"));
            }
        }

        @Override
        public void onError(Throwable throwable) {
            callback.onError(throwable);
        }

        @Override
        public void onCompleted() {
        }
    });
}