Examples with OperationBatch com.alibaba.graphscope.groot.operation.OperationBatch used on opensource projects

Example 1 with OperationBatch

use of com.alibaba.graphscope.groot.operation.OperationBatch in project GraphScope by alibaba.

the class GraphWriter method writeBatch.

public void writeBatch(String requestId, String writeSession, List<WriteRequest> writeRequests, CompletionCallback<Long> callback) {
    this.pendingWriteCount.incrementAndGet();
    GraphSchema schema = snapshotCache.getSnapshotWithSchema().getGraphDef();
    OperationBatch.Builder batchBuilder = OperationBatch.newBuilder();
    for (WriteRequest writeRequest : writeRequests) {
        OperationType operationType = writeRequest.getOperationType();
        DataRecord dataRecord = writeRequest.getDataRecord();
        switch(operationType) {
            case OVERWRITE_VERTEX:
                addOverwriteVertexOperation(batchBuilder, schema, dataRecord);
                break;
            case UPDATE_VERTEX:
                addUpdateVertexOperation(batchBuilder, schema, dataRecord);
                break;
            case DELETE_VERTEX:
                addDeleteVertexOperation(batchBuilder, schema, dataRecord);
                break;
            case OVERWRITE_EDGE:
                addOverwriteEdgeOperation(batchBuilder, schema, dataRecord);
                break;
            case UPDATE_EDGE:
                addUpdateEdgeOperation(batchBuilder, schema, dataRecord);
                break;
            case DELETE_EDGE:
                addDeleteEdgeOperation(batchBuilder, schema, dataRecord);
                break;
            default:
                throw new IllegalArgumentException("Invalid operationType [" + operationType + "]");
        }
    }
    OperationBatch operationBatch = batchBuilder.build();
    int writeQueueId = getWriteQueueId(writeSession);
    int ingestorId = this.metaService.getIngestorIdForQueue(writeQueueId);
    long startTimeNano = System.nanoTime();
    this.ingestWriteClients.getClient(ingestorId).writeIngestorAsync(requestId, writeQueueId, operationBatch, new CompletionCallback<Long>() {

        @Override
        public void onCompleted(Long res) {
            long writeSnapshotId = res;
            lastWrittenSnapshotId.updateAndGet(x -> x < writeSnapshotId ? writeSnapshotId : x);
            writeRequestsTotal.addAndGet(writeRequests.size());
            finish();
            callback.onCompleted(res);
        }

        @Override
        public void onError(Throwable t) {
            finish();
            callback.onError(t);
        }

        void finish() {
            long ingestorCompleteTimeNano = System.nanoTime();
            ingestorBlockTimeNano.addAndGet(ingestorCompleteTimeNano - startTimeNano);
            pendingWriteCount.decrementAndGet();
        }
    });
}

Example 2 with OperationBatch

use of com.alibaba.graphscope.groot.operation.OperationBatch in project GraphScope by alibaba.

the class BatchSender method asyncSendWithRetry.

public void asyncSendWithRetry(String requestId, int queueId, long snapshotId, long offset, OperationBatch operationBatch) {
    int partitionCount = metaService.getPartitionCount();
    Map<Integer, Builder> storeToBatchBuilder = new HashMap<>();
    Function<Integer, Builder> storeDataBatchBuilderFunc = k -> StoreDataBatch.newBuilder().requestId(requestId).queueId(queueId).snapshotId(snapshotId).offset(offset);
    for (OperationBlob operationBlob : operationBatch) {
        long partitionKey = operationBlob.getPartitionKey();
        if (partitionKey == -1L) {
            // replicate to all store node
            for (int i = 0; i < this.storeCount; i++) {
                StoreDataBatch.Builder batchBuilder = storeToBatchBuilder.computeIfAbsent(i, storeDataBatchBuilderFunc);
                batchBuilder.addOperation(-1, operationBlob);
            }
        } else {
            int partitionId = PartitionUtils.getPartitionIdFromKey(partitionKey, partitionCount);
            int storeId = metaService.getStoreIdByPartition(partitionId);
            StoreDataBatch.Builder batchBuilder = storeToBatchBuilder.computeIfAbsent(storeId, storeDataBatchBuilderFunc);
            batchBuilder.addOperation(partitionId, operationBlob);
        }
    }
    storeToBatchBuilder.forEach((storeId, batchBuilder) -> {
        while (!shouldStop) {
            try {
                storeSendBuffer.get(storeId).put(batchBuilder.build());
                break;
            } catch (InterruptedException e) {
                logger.warn("send buffer interrupted", e);
            }
        }
    });
}

Example 3 with OperationBatch

use of com.alibaba.graphscope.groot.operation.OperationBatch in project GraphScope by alibaba.

the class SchemaManager method submitBatchDdl.

public void submitBatchDdl(String requestId, String sessionId, DdlRequestBatch ddlRequestBatch, CompletionCallback<Long> callback) {
    logger.info("submitBatchDdl requestId [" + requestId + "], sessionId [" + sessionId + "]");
    if (!ready) {
        callback.onError(new IllegalStateException("SchemaManager is recovering"));
        return;
    }
    this.singleThreadExecutor.execute(() -> {
        try {
            if (!ready) {
                callback.onError(new IllegalStateException("SchemaManager is recovering"));
                return;
            }
            GraphDef tmpGraphDef = this.graphDefRef.get();
            DdlResult ddlResult = this.ddlExecutors.executeDdlRequestBatch(ddlRequestBatch, tmpGraphDef, this.partitionCount);
            GraphDef graphDefResult = ddlResult.getGraphDef();
            List<Operation> ddlOperations = ddlResult.getDdlOperations();
            this.snapshotManager.lockWriteSnapshot();
            BatchId batchId;
            try {
                long currentWriteSnapshotId = this.snapshotManager.getCurrentWriteSnapshotId();
                OperationBatch operationBatch = OperationBatch.newBuilder(ddlOperations).setLatestSnapshotId(currentWriteSnapshotId).build();
                batchId = this.ddlWriter.writeOperations(requestId, operationBatch);
            } finally {
                this.snapshotManager.unlockWriteSnapshot();
            }
            long snapshotId = batchId.getSnapshotId();
            CompletableFuture<Void> future = new CompletableFuture<>();
            this.snapshotManager.addSnapshotListener(snapshotId, () -> {
                this.graphDefRef.set(graphDefResult);
                future.complete(null);
            });
            future.get();
            callback.onCompleted(snapshotId);
        } catch (Exception e) {
            logger.error("Error in Ddl requestId [" + requestId + "], sessionId [" + sessionId + "]", e);
            this.ready = false;
            callback.onError(e);
            this.singleThreadExecutor.execute(() -> recover());
        }
    });
}

Example 4 with OperationBatch

use of com.alibaba.graphscope.groot.operation.OperationBatch in project GraphScope by alibaba.

the class StoreService method writeStore.

private Map<Integer, OperationBatch> writeStore(long snapshotId, Map<Integer, OperationBatch> partitionToBatch, AtomicBoolean hasDdl) throws ExecutionException, InterruptedException {
    Map<Integer, OperationBatch> batchNeedRetry = new ConcurrentHashMap<>();
    AtomicInteger counter = new AtomicInteger(partitionToBatch.size());
    CompletableFuture<Object> future = new CompletableFuture<>();
    for (Map.Entry<Integer, OperationBatch> e : partitionToBatch.entrySet()) {
        int partitionId = e.getKey();
        OperationBatch batch = e.getValue();
        logger.debug("writeStore partition [" + partitionId + "]");
        this.writeExecutor.execute(() -> {
            try {
                if (partitionId != -1) {
                    // Ignore Marker
                    // Only support partition operation for now
                    long beforeWriteTime = System.nanoTime();
                    GraphPartition partition = this.idToPartition.get(partitionId);
                    if (partition == null) {
                        throw new IllegalStateException("partition [" + partitionId + "] is not initialized / exists");
                    }
                    if (partition.writeBatch(snapshotId, batch)) {
                        hasDdl.set(true);
                    }
                    long afterWriteTime = System.nanoTime();
                    this.partitionToMetric.get(partitionId).add(afterWriteTime - beforeWriteTime);
                }
            } catch (Exception ex) {
                logger.error("write to partition [" + partitionId + "] failed, snapshotId [" + snapshotId + "]. will retry", ex);
                batchNeedRetry.put(partitionId, batch);
            }
            if (counter.decrementAndGet() == 0) {
                future.complete(null);
            }
        });
    }
    future.get();
    if (batchNeedRetry.size() > 0) {
        try {
            Thread.sleep(1000L);
        } catch (InterruptedException e) {
        // Ignore
        }
    }
    return batchNeedRetry;
}

Example 5 with OperationBatch

use of com.alibaba.graphscope.groot.operation.OperationBatch in project GraphScope by alibaba.

the class StoreService method batchWrite.

public boolean batchWrite(StoreDataBatch storeDataBatch) throws ExecutionException, InterruptedException {
    long snapshotId = storeDataBatch.getSnapshotId();
    List<Map<Integer, OperationBatch>> dataBatch = storeDataBatch.getDataBatch();
    AtomicBoolean hasDdl = new AtomicBoolean(false);
    for (Map<Integer, OperationBatch> partitionToBatch : dataBatch) {
        while (!shouldStop && partitionToBatch.size() != 0) {
            partitionToBatch = writeStore(snapshotId, partitionToBatch, hasDdl);
        }
    }
    return hasDdl.get();
}